From a94391d9660727a8c8875193b856390c7bdd8bd4 Mon Sep 17 00:00:00 2001 From: "abhishek.sharma" Date: Sat, 20 Dec 2025 11:47:03 +0530 Subject: [PATCH 01/33] Fix model training state restoration in GRPO trainer Store the model's training state before generation and restore inference mode after completion if the model wasn't originally in training mode. This ensures the model returns to the correct state after generate and score operations. --- unsloth/models/rl_replacements.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.py index 7d4d520c1f..dd139ffd25 100644 --- a/unsloth/models/rl_replacements.py +++ b/unsloth/models/rl_replacements.py @@ -259,6 +259,7 @@ def grpo_trainer__generate_and_score_completions(function_name, function): # The new multi-line string that will replace the line above replacement_lines = """ batch_size = self.args.per_device_train_batch_size if mode == "train" else self.args.per_device_eval_batch_size + _was_training = self.model.training try: # TRL 0.23.1 and below path if not has_images: @@ -387,6 +388,13 @@ def grpo_trainer__generate_and_score_completions(function_name, function): patched = patched[: match.start()] + wrapped + patched[match.end() :] function = patched + + function = function.replace( + " return output", # 8 spaces before 'return' + """ if not _was_training: + self.model.for_inference() + return output""" + ) return function From bef0371cc668f48c0cbc8dd3a629b05dae6754d5 Mon Sep 17 00:00:00 2001 From: numb3r33 Date: Sat, 20 Dec 2025 12:30:33 +0530 Subject: [PATCH 02/33] Remove the comment. --- unsloth/models/rl_replacements.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.py index dd139ffd25..248c5aab85 100644 --- a/unsloth/models/rl_replacements.py +++ b/unsloth/models/rl_replacements.py @@ -390,7 +390,7 @@ def grpo_trainer__generate_and_score_completions(function_name, function): function = patched function = function.replace( - " return output", # 8 spaces before 'return' + " return output", """ if not _was_training: self.model.for_inference() return output""" From 7620e75c3964ca60a7a2174bc9b42314d61f9b42 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 20 Dec 2025 07:02:30 +0000 Subject: [PATCH 03/33] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- unsloth/models/rl_replacements.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.py index 248c5aab85..e13e5e6d78 100644 --- a/unsloth/models/rl_replacements.py +++ b/unsloth/models/rl_replacements.py @@ -388,12 +388,12 @@ def grpo_trainer__generate_and_score_completions(function_name, function): patched = patched[: match.start()] + wrapped + patched[match.end() :] function = patched - + function = function.replace( - " return output", - """ if not _was_training: + " return output", + """ if not _was_training: self.model.for_inference() - return output""" + return output""", ) return function From 89f2d3a28b0744edc5476eb2341ee91bd4621a81 Mon Sep 17 00:00:00 2001 From: numb3r33 Date: Wed, 24 Dec 2025 00:14:50 +0530 Subject: [PATCH 04/33] Fix indentation handling in grpo_trainer return statement replacement Use regex to dynamically detect and preserve the original indentation when replacing the 'return output' statement, instead of hardcoding spaces. This ensures the patched code maintains consistent indentation regardless of the original formatting. --- unsloth/models/rl_replacements.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.py index e13e5e6d78..bcac699b3f 100644 --- a/unsloth/models/rl_replacements.py +++ b/unsloth/models/rl_replacements.py @@ -388,13 +388,17 @@ def grpo_trainer__generate_and_score_completions(function_name, function): patched = patched[: match.start()] + wrapped + patched[match.end() :] function = patched + + match = re.search(r'^(\s*)return output', function, re.MULTILINE) - function = function.replace( - " return output", - """ if not _was_training: - self.model.for_inference() - return output""", - ) + if match: + indent = match.group(1) + function = function.replace( + f"{indent}return output", + f"""{indent}if not _was_training: + {indent} self.model.for_inference() + {indent}return output""" + ) return function From ce7251458ecb1c966b3ce1ac9f3b4a3bba0ce7b7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 23 Dec 2025 19:20:08 +0000 Subject: [PATCH 05/33] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- unsloth/models/rl_replacements.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.py index bcac699b3f..5158019132 100644 --- a/unsloth/models/rl_replacements.py +++ b/unsloth/models/rl_replacements.py @@ -388,8 +388,8 @@ def grpo_trainer__generate_and_score_completions(function_name, function): patched = patched[: match.start()] + wrapped + patched[match.end() :] function = patched - - match = re.search(r'^(\s*)return output', function, re.MULTILINE) + + match = re.search(r"^(\s*)return output", function, re.MULTILINE) if match: indent = match.group(1) @@ -397,7 +397,7 @@ def grpo_trainer__generate_and_score_completions(function_name, function): f"{indent}return output", f"""{indent}if not _was_training: {indent} self.model.for_inference() - {indent}return output""" + {indent}return output""", ) return function From 880fc3d1756a6c98db19232142e6e087febed599 Mon Sep 17 00:00:00 2001 From: numb3r33 Date: Wed, 24 Dec 2025 01:02:03 +0530 Subject: [PATCH 06/33] Refactor return statement replacement to use explicit newlines Replace f-string triple-quoted approach with explicit newline characters for clearer string construction in the grpo_trainer patch. --- unsloth/models/rl_replacements.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.py index 5158019132..8436ca0dc9 100644 --- a/unsloth/models/rl_replacements.py +++ b/unsloth/models/rl_replacements.py @@ -393,12 +393,8 @@ def grpo_trainer__generate_and_score_completions(function_name, function): if match: indent = match.group(1) - function = function.replace( - f"{indent}return output", - f"""{indent}if not _was_training: - {indent} self.model.for_inference() - {indent}return output""", - ) + new_code = indent + "if not _was_training:\n" + indent + " self.model.for_inference()\n" + indent + "return output" + function = function.replace(f"{indent}return output", new_code) return function From e7d04737dd3732c7c545147fddce6ed025bfa5a3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 23 Dec 2025 20:06:36 +0000 Subject: [PATCH 07/33] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- unsloth/models/rl_replacements.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.py index 8436ca0dc9..f0f0386bd1 100644 --- a/unsloth/models/rl_replacements.py +++ b/unsloth/models/rl_replacements.py @@ -393,7 +393,14 @@ def grpo_trainer__generate_and_score_completions(function_name, function): if match: indent = match.group(1) - new_code = indent + "if not _was_training:\n" + indent + " self.model.for_inference()\n" + indent + "return output" + new_code = ( + indent + + "if not _was_training:\n" + + indent + + " self.model.for_inference()\n" + + indent + + "return output" + ) function = function.replace(f"{indent}return output", new_code) return function From b5addbc936933ad3ca682a0cc2f3eeececfba321 Mon Sep 17 00:00:00 2001 From: Kaitao Yang Date: Sun, 4 Jan 2026 09:21:44 -0800 Subject: [PATCH 08/33] remove unused variable BlockDiagonalCausalMask --- unsloth/utils/attention_dispatch.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/unsloth/utils/attention_dispatch.py b/unsloth/utils/attention_dispatch.py index 0e5f3c1951..a7620549be 100644 --- a/unsloth/utils/attention_dispatch.py +++ b/unsloth/utils/attention_dispatch.py @@ -32,9 +32,6 @@ from ..utils.packing import ( if HAS_FLASH_ATTENTION: from flash_attn import flash_attn_func, flash_attn_varlen_func HAS_XFORMERS = xformers is not None -BlockDiagonalCausalMask = None -if HAS_XFORMERS: - BlockDiagonalCausalMask = xformers.attn_bias.BlockDiagonalCausalMask SDPA_HAS_GQA = "enable_gqa" in (scaled_dot_product_attention.__doc__ or "") FLASH_VARLEN = "flash_varlen" From 6c6d0dfef1bce443b2030dd46f04e9fcbb981dff Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 5 Jan 2026 05:02:53 +0000 Subject: [PATCH 09/33] Fix vLLM PDL bug on Blackwell GPUs (B200/B100) vLLM's LoRA Triton kernels use tl.extra.cuda.gdc_wait() for PDL optimization on SM90+ GPUs. This fails on SM100 (Blackwell) during CUDA graph capture because Triton's pipeliner cannot handle gdc_wait in complex kernels. This fix: - Detects SM100 GPUs and applies the workaround automatically - Sets TRITON_DISABLE_PDL=1 environment variable - Monkey-patches supports_pdl to return False in lora_expand_op and lora_shrink_op - Checks GitHub issue #30872 status (with 3s timeout) to auto-disable the workaround once the upstream fix is merged - Includes quick internet connectivity check (0.5s) to avoid delays when offline Fixes the error: 'tt.elementwise_inline_asm' op pipeliner doesn't know how to predicate this op LLVM ERROR: Fatal pipeliner error See: https://github.com/vllm-project/vllm/issues/30872 --- unsloth/__init__.py | 3 + unsloth/import_fixes.py | 121 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 124 insertions(+) diff --git a/unsloth/__init__.py b/unsloth/__init__.py index d9633e8ec1..86fb00fe0e 100644 --- a/unsloth/__init__.py +++ b/unsloth/__init__.py @@ -126,6 +126,7 @@ from .import_fixes import ( fix_xformers_performance_issue, fix_vllm_aimv2_issue, fix_vllm_guided_decoding_params, + fix_vllm_pdl_blackwell, ignore_logger_messages, patch_ipykernel_hf_xet, patch_trackio, @@ -138,6 +139,7 @@ from .import_fixes import ( fix_xformers_performance_issue() fix_vllm_aimv2_issue() fix_vllm_guided_decoding_params() +fix_vllm_pdl_blackwell() ignore_logger_messages() patch_ipykernel_hf_xet() patch_trackio() @@ -149,6 +151,7 @@ fix_executorch() del fix_xformers_performance_issue del fix_vllm_aimv2_issue del fix_vllm_guided_decoding_params +del fix_vllm_pdl_blackwell del ignore_logger_messages del patch_ipykernel_hf_xet del patch_trackio diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index bb6996a3e3..91ba35e21e 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -556,3 +556,124 @@ def fix_huggingface_hub(): huggingface_hub.is_offline_mode = ( lambda: huggingface_hub.constants.HF_HUB_OFFLINE ) + + +def fix_vllm_pdl_blackwell(): + """ + Fix vLLM PDL (Programmatic Dependent Launch) bug on Blackwell GPUs (SM100). + + The issue: vLLM's LoRA Triton kernels use tl.extra.cuda.gdc_wait() for PDL + optimization on SM90+ GPUs. This fails on SM100 (B200/B100) during CUDA graph + capture because Triton's pipeliner can't handle gdc_wait in complex kernels. + + See: https://github.com/vllm-project/vllm/issues/30872 + """ + if importlib.util.find_spec("vllm") is None: + return + + # Check if we have a CUDA GPU + try: + import torch + if not torch.cuda.is_available(): + return + major, minor = torch.cuda.get_device_capability() + except Exception: + return + + # Only SM100 (Blackwell) is affected - SM90 (Hopper) works fine + if major != 10: + return + + gpu_name = torch.cuda.get_device_name() + + # Check if vLLM has the PDL-related modules before doing internet check + try: + has_expand_op = importlib.util.find_spec("vllm.lora.ops.triton_ops.lora_expand_op") is not None + except (ModuleNotFoundError, ValueError): + has_expand_op = False + try: + has_shrink_op = importlib.util.find_spec("vllm.lora.ops.triton_ops.lora_shrink_op") is not None + except (ModuleNotFoundError, ValueError): + has_shrink_op = False + if not has_expand_op and not has_shrink_op: + # Old vLLM version without PDL support - just set env var to be safe + os.environ["TRITON_DISABLE_PDL"] = "1" + logger.info( + f"Unsloth: Set TRITON_DISABLE_PDL=1 for SM{major}{minor} ({gpu_name}) - " + f"vLLM PDL modules not found" + ) + return + + # Check if GitHub issue is closed (fix merged upstream) + issue_closed = False + try: + import socket + import urllib.request + import json as json_module + + # Quick internet connectivity check (0.5s timeout) + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(0.5) + try: + sock.connect(("api.github.com", 443)) + has_internet = True + except (socket.timeout, OSError): + has_internet = False + finally: + sock.close() + + if has_internet: + api_url = "https://api.github.com/repos/vllm-project/vllm/issues/30872" + req = urllib.request.Request( + api_url, + headers={ + "User-Agent": "Unsloth-PDL-Fix", + "Accept": "application/vnd.github.v3+json", + } + ) + with urllib.request.urlopen(req, timeout=3) as response: + data = json_module.loads(response.read().decode()) + issue_closed = data.get("state") == "closed" + except Exception: + # If we can't check, assume issue is still open (apply fix to be safe) + pass + + if issue_closed: + logger.info( + f"Unsloth: SM{major}{minor} ({gpu_name}) detected but PDL issue #30872 " + f"is closed - skipping PDL fix" + ) + return + + # Apply the PDL fix + os.environ["TRITON_DISABLE_PDL"] = "1" + + def fake_supports_pdl(device=None): + return False + + patched = [] + + try: + import vllm.lora.ops.triton_ops.lora_expand_op as expand_op + expand_op.supports_pdl = fake_supports_pdl + patched.append("lora_expand_op") + except (ImportError, ModuleNotFoundError, AttributeError): + pass + + try: + import vllm.lora.ops.triton_ops.lora_shrink_op as shrink_op + shrink_op.supports_pdl = fake_supports_pdl + patched.append("lora_shrink_op") + except (ImportError, ModuleNotFoundError, AttributeError): + pass + + if patched: + logger.info( + f"Unsloth: Applied PDL fix for SM{major}{minor} ({gpu_name}) - " + f"patched: {', '.join(patched)}" + ) + else: + # Just set the env var - vLLM might be an older version without supports_pdl + logger.info( + f"Unsloth: Set TRITON_DISABLE_PDL=1 for SM{major}{minor} ({gpu_name})" + ) From efe949c941a752e3f1872b84d2fc9b1b54e9358c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 5 Jan 2026 05:03:28 +0000 Subject: [PATCH 10/33] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- unsloth/import_fixes.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index 91ba35e21e..e8c5e16df4 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -574,6 +574,7 @@ def fix_vllm_pdl_blackwell(): # Check if we have a CUDA GPU try: import torch + if not torch.cuda.is_available(): return major, minor = torch.cuda.get_device_capability() @@ -588,11 +589,17 @@ def fix_vllm_pdl_blackwell(): # Check if vLLM has the PDL-related modules before doing internet check try: - has_expand_op = importlib.util.find_spec("vllm.lora.ops.triton_ops.lora_expand_op") is not None + has_expand_op = ( + importlib.util.find_spec("vllm.lora.ops.triton_ops.lora_expand_op") + is not None + ) except (ModuleNotFoundError, ValueError): has_expand_op = False try: - has_shrink_op = importlib.util.find_spec("vllm.lora.ops.triton_ops.lora_shrink_op") is not None + has_shrink_op = ( + importlib.util.find_spec("vllm.lora.ops.triton_ops.lora_shrink_op") + is not None + ) except (ModuleNotFoundError, ValueError): has_shrink_op = False if not has_expand_op and not has_shrink_op: @@ -626,12 +633,12 @@ def fix_vllm_pdl_blackwell(): api_url = "https://api.github.com/repos/vllm-project/vllm/issues/30872" req = urllib.request.Request( api_url, - headers={ + headers = { "User-Agent": "Unsloth-PDL-Fix", "Accept": "application/vnd.github.v3+json", - } + }, ) - with urllib.request.urlopen(req, timeout=3) as response: + with urllib.request.urlopen(req, timeout = 3) as response: data = json_module.loads(response.read().decode()) issue_closed = data.get("state") == "closed" except Exception: @@ -648,13 +655,14 @@ def fix_vllm_pdl_blackwell(): # Apply the PDL fix os.environ["TRITON_DISABLE_PDL"] = "1" - def fake_supports_pdl(device=None): + def fake_supports_pdl(device = None): return False patched = [] try: import vllm.lora.ops.triton_ops.lora_expand_op as expand_op + expand_op.supports_pdl = fake_supports_pdl patched.append("lora_expand_op") except (ImportError, ModuleNotFoundError, AttributeError): @@ -662,6 +670,7 @@ def fix_vllm_pdl_blackwell(): try: import vllm.lora.ops.triton_ops.lora_shrink_op as shrink_op + shrink_op.supports_pdl = fake_supports_pdl patched.append("lora_shrink_op") except (ImportError, ModuleNotFoundError, AttributeError): From 36c9a841eb959f279b0170041f86e43c7421b514 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 5 Jan 2026 05:03:56 +0000 Subject: [PATCH 11/33] Sync chat_template from tokenizer to vLLM When using base models with custom chat templates applied after loading, vLLM's internal tokenizer may not have the chat_template set. This causes issues during RL training with vLLM inference. This fix syncs the chat_template from the processing_class (the tokenizer you loaded and configured) to vLLM's internal tokenizer during trainer initialization, but only if vLLM's tokenizer does not already have one set. --- unsloth/models/rl.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py index 88aeeda8a1..20dafaaaa4 100644 --- a/unsloth/models/rl.py +++ b/unsloth/models/rl.py @@ -694,6 +694,20 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"): ) RLTrainer_post += training_check + # Sync chat_template from processing_class to vLLM's tokenizer + # This fixes base models that have custom chat templates applied after loading + if "model" in call_args: + vllm_chat_template_sync = ( + "if hasattr(self, 'llm') and self.llm is not None and hasattr(self.llm, 'get_tokenizer'):\n" + " _vllm_tok = self.llm.get_tokenizer()\n" + " _pc = getattr(self, 'processing_class', None)\n" + " if _pc is not None and getattr(_pc, 'chat_template', None) is not None:\n" + " if _vllm_tok.chat_template is None:\n" + " _vllm_tok.chat_template = _pc.chat_template\n" + "pass\n" + ) + RLTrainer_post += vllm_chat_template_sync + # Edit optional metrics other_metrics_processor = "" if trainer_file in RL_METRICS_CHANGES: From fbdb3b524e93ea99d8845696e9c40ce64bef349d Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 5 Jan 2026 05:10:24 +0000 Subject: [PATCH 12/33] Add tokenizer fallback for chat_template sync --- unsloth/models/rl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py index 20dafaaaa4..b75ae383db 100644 --- a/unsloth/models/rl.py +++ b/unsloth/models/rl.py @@ -700,7 +700,7 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"): vllm_chat_template_sync = ( "if hasattr(self, 'llm') and self.llm is not None and hasattr(self.llm, 'get_tokenizer'):\n" " _vllm_tok = self.llm.get_tokenizer()\n" - " _pc = getattr(self, 'processing_class', None)\n" + " _pc = getattr(self, 'processing_class', None) or getattr(self, 'tokenizer', None)\n" " if _pc is not None and getattr(_pc, 'chat_template', None) is not None:\n" " if _vllm_tok.chat_template is None:\n" " _vllm_tok.chat_template = _pc.chat_template\n" From 227c31f0caf3cb30a06a77c0a1b010cc7e09007d Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 5 Jan 2026 05:24:52 +0000 Subject: [PATCH 13/33] Address review feedback: refactor and scan all GPUs - Add _spec_exists helper function to reduce duplication - Scan all GPUs for SM100 instead of just device 0 - Use loop for module patching to improve maintainability --- unsloth/import_fixes.py | 85 ++++++++++++++++++++--------------------- 1 file changed, 41 insertions(+), 44 deletions(-) diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index e8c5e16df4..7d368d027a 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -571,42 +571,44 @@ def fix_vllm_pdl_blackwell(): if importlib.util.find_spec("vllm") is None: return - # Check if we have a CUDA GPU + # Check if any CUDA GPU is SM100 (Blackwell) try: import torch if not torch.cuda.is_available(): return - major, minor = torch.cuda.get_device_capability() + + # Scan all GPUs for SM100 - fix applies globally via env var and monkey-patch + has_sm100 = False + sm100_gpu_name = None + for i in range(torch.cuda.device_count()): + major, minor = torch.cuda.get_device_capability(i) + if major == 10: + has_sm100 = True + sm100_gpu_name = torch.cuda.get_device_name(i) + break + + if not has_sm100: + return except Exception: return - # Only SM100 (Blackwell) is affected - SM90 (Hopper) works fine - if major != 10: - return - - gpu_name = torch.cuda.get_device_name() + # Helper to check if module spec exists + def _spec_exists(name): + try: + return importlib.util.find_spec(name) is not None + except (ModuleNotFoundError, ValueError): + return False # Check if vLLM has the PDL-related modules before doing internet check - try: - has_expand_op = ( - importlib.util.find_spec("vllm.lora.ops.triton_ops.lora_expand_op") - is not None - ) - except (ModuleNotFoundError, ValueError): - has_expand_op = False - try: - has_shrink_op = ( - importlib.util.find_spec("vllm.lora.ops.triton_ops.lora_shrink_op") - is not None - ) - except (ModuleNotFoundError, ValueError): - has_shrink_op = False + has_expand_op = _spec_exists("vllm.lora.ops.triton_ops.lora_expand_op") + has_shrink_op = _spec_exists("vllm.lora.ops.triton_ops.lora_shrink_op") + if not has_expand_op and not has_shrink_op: # Old vLLM version without PDL support - just set env var to be safe os.environ["TRITON_DISABLE_PDL"] = "1" logger.info( - f"Unsloth: Set TRITON_DISABLE_PDL=1 for SM{major}{minor} ({gpu_name}) - " + f"Unsloth: Set TRITON_DISABLE_PDL=1 for SM100 ({sm100_gpu_name}) - " f"vLLM PDL modules not found" ) return @@ -633,12 +635,12 @@ def fix_vllm_pdl_blackwell(): api_url = "https://api.github.com/repos/vllm-project/vllm/issues/30872" req = urllib.request.Request( api_url, - headers = { + headers={ "User-Agent": "Unsloth-PDL-Fix", "Accept": "application/vnd.github.v3+json", }, ) - with urllib.request.urlopen(req, timeout = 3) as response: + with urllib.request.urlopen(req, timeout=3) as response: data = json_module.loads(response.read().decode()) issue_closed = data.get("state") == "closed" except Exception: @@ -647,7 +649,7 @@ def fix_vllm_pdl_blackwell(): if issue_closed: logger.info( - f"Unsloth: SM{major}{minor} ({gpu_name}) detected but PDL issue #30872 " + f"Unsloth: SM100 ({sm100_gpu_name}) detected but PDL issue #30872 " f"is closed - skipping PDL fix" ) return @@ -655,34 +657,29 @@ def fix_vllm_pdl_blackwell(): # Apply the PDL fix os.environ["TRITON_DISABLE_PDL"] = "1" - def fake_supports_pdl(device = None): + def fake_supports_pdl(device=None): return False patched = [] - - try: - import vllm.lora.ops.triton_ops.lora_expand_op as expand_op - - expand_op.supports_pdl = fake_supports_pdl - patched.append("lora_expand_op") - except (ImportError, ModuleNotFoundError, AttributeError): - pass - - try: - import vllm.lora.ops.triton_ops.lora_shrink_op as shrink_op - - shrink_op.supports_pdl = fake_supports_pdl - patched.append("lora_shrink_op") - except (ImportError, ModuleNotFoundError, AttributeError): - pass + modules_to_patch = { + "lora_expand_op": "vllm.lora.ops.triton_ops.lora_expand_op", + "lora_shrink_op": "vllm.lora.ops.triton_ops.lora_shrink_op", + } + for name, path in modules_to_patch.items(): + try: + module = importlib.import_module(path) + module.supports_pdl = fake_supports_pdl + patched.append(name) + except (ImportError, ModuleNotFoundError, AttributeError): + pass if patched: logger.info( - f"Unsloth: Applied PDL fix for SM{major}{minor} ({gpu_name}) - " + f"Unsloth: Applied PDL fix for SM100 ({sm100_gpu_name}) - " f"patched: {', '.join(patched)}" ) else: # Just set the env var - vLLM might be an older version without supports_pdl logger.info( - f"Unsloth: Set TRITON_DISABLE_PDL=1 for SM{major}{minor} ({gpu_name})" + f"Unsloth: Set TRITON_DISABLE_PDL=1 for SM100 ({sm100_gpu_name})" ) From eac1f6b0101ce12ae3d630160f9e3d8593a70779 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 5 Jan 2026 05:24:59 +0000 Subject: [PATCH 14/33] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- unsloth/import_fixes.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index 7d368d027a..77693d4cf3 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -635,12 +635,12 @@ def fix_vllm_pdl_blackwell(): api_url = "https://api.github.com/repos/vllm-project/vllm/issues/30872" req = urllib.request.Request( api_url, - headers={ + headers = { "User-Agent": "Unsloth-PDL-Fix", "Accept": "application/vnd.github.v3+json", }, ) - with urllib.request.urlopen(req, timeout=3) as response: + with urllib.request.urlopen(req, timeout = 3) as response: data = json_module.loads(response.read().decode()) issue_closed = data.get("state") == "closed" except Exception: @@ -657,7 +657,7 @@ def fix_vllm_pdl_blackwell(): # Apply the PDL fix os.environ["TRITON_DISABLE_PDL"] = "1" - def fake_supports_pdl(device=None): + def fake_supports_pdl(device = None): return False patched = [] @@ -680,6 +680,4 @@ def fix_vllm_pdl_blackwell(): ) else: # Just set the env var - vLLM might be an older version without supports_pdl - logger.info( - f"Unsloth: Set TRITON_DISABLE_PDL=1 for SM100 ({sm100_gpu_name})" - ) + logger.info(f"Unsloth: Set TRITON_DISABLE_PDL=1 for SM100 ({sm100_gpu_name})") From ba548ff8c22b055c5acaa02eb0d67c2e238413ce Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 5 Jan 2026 05:25:53 +0000 Subject: [PATCH 15/33] Combine nested if statements for clarity --- unsloth/models/rl.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py index b75ae383db..e1ecd6df2f 100644 --- a/unsloth/models/rl.py +++ b/unsloth/models/rl.py @@ -701,9 +701,8 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"): "if hasattr(self, 'llm') and self.llm is not None and hasattr(self.llm, 'get_tokenizer'):\n" " _vllm_tok = self.llm.get_tokenizer()\n" " _pc = getattr(self, 'processing_class', None) or getattr(self, 'tokenizer', None)\n" - " if _pc is not None and getattr(_pc, 'chat_template', None) is not None:\n" - " if _vllm_tok.chat_template is None:\n" - " _vllm_tok.chat_template = _pc.chat_template\n" + " if _pc is not None and getattr(_pc, 'chat_template', None) is not None and _vllm_tok.chat_template is None:\n" + " _vllm_tok.chat_template = _pc.chat_template\n" "pass\n" ) RLTrainer_post += vllm_chat_template_sync From 35219633ab161f062a826f84b037ac18f6390e7e Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 5 Jan 2026 06:53:42 +0000 Subject: [PATCH 16/33] Fix PDL patch: target utils.py source module and clear lru_cache - Patch vllm.lora.ops.triton_ops.utils directly where supports_pdl is defined - Clear lru_cache before patching to prevent stale cached results - Add fused_moe_lora_op to consumer modules list - Use *args, **kwargs in fake function for compatibility --- unsloth/import_fixes.py | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index 77693d4cf3..469674b29e 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -601,10 +601,11 @@ def fix_vllm_pdl_blackwell(): return False # Check if vLLM has the PDL-related modules before doing internet check + has_utils = _spec_exists("vllm.lora.ops.triton_ops.utils") has_expand_op = _spec_exists("vllm.lora.ops.triton_ops.lora_expand_op") has_shrink_op = _spec_exists("vllm.lora.ops.triton_ops.lora_shrink_op") - if not has_expand_op and not has_shrink_op: + if not has_utils and not has_expand_op and not has_shrink_op: # Old vLLM version without PDL support - just set env var to be safe os.environ["TRITON_DISABLE_PDL"] = "1" logger.info( @@ -657,19 +658,39 @@ def fix_vllm_pdl_blackwell(): # Apply the PDL fix os.environ["TRITON_DISABLE_PDL"] = "1" - def fake_supports_pdl(device = None): + def fake_supports_pdl(*args, **kwargs): return False patched = [] - modules_to_patch = { + + # First, patch the source module (utils.py) where supports_pdl is defined. + # This is critical because supports_pdl uses @lru_cache - we must clear the + # cache to prevent stale cached results from the original function. + try: + utils_module = importlib.import_module("vllm.lora.ops.triton_ops.utils") + if hasattr(utils_module, "supports_pdl"): + original_fn = utils_module.supports_pdl + if hasattr(original_fn, "cache_clear"): + original_fn.cache_clear() + utils_module.supports_pdl = fake_supports_pdl + patched.append("utils") + except (ImportError, ModuleNotFoundError, AttributeError): + pass + + # Also patch the consumer modules that import supports_pdl from utils. + # This ensures the patched function is used even if the module was already + # imported before this fix runs. + consumer_modules = { "lora_expand_op": "vllm.lora.ops.triton_ops.lora_expand_op", "lora_shrink_op": "vllm.lora.ops.triton_ops.lora_shrink_op", + "fused_moe_lora_op": "vllm.lora.ops.triton_ops.fused_moe_lora_op", } - for name, path in modules_to_patch.items(): + for name, path in consumer_modules.items(): try: module = importlib.import_module(path) - module.supports_pdl = fake_supports_pdl - patched.append(name) + if hasattr(module, "supports_pdl"): + module.supports_pdl = fake_supports_pdl + patched.append(name) except (ImportError, ModuleNotFoundError, AttributeError): pass From b9bbf4771002ce67b119c8f1ebe4eb0b9087866e Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Mon, 5 Jan 2026 07:02:36 +0000 Subject: [PATCH 17/33] Improve TRL compatibility and GRPO state restore --- unsloth/kernels/cross_entropy_loss.py | 2 +- unsloth/models/cohere.py | 4 +- unsloth/models/gemma.py | 4 +- unsloth/models/gemma2.py | 4 +- unsloth/models/granite.py | 4 +- unsloth/models/loader_utils.py | 1 - unsloth/models/rl.py | 65 ++++++++++++++++++++++++--- unsloth/trainer.py | 4 +- 8 files changed, 67 insertions(+), 21 deletions(-) diff --git a/unsloth/kernels/cross_entropy_loss.py b/unsloth/kernels/cross_entropy_loss.py index 912e6f7e3f..fbb14013ff 100644 --- a/unsloth/kernels/cross_entropy_loss.py +++ b/unsloth/kernels/cross_entropy_loss.py @@ -24,7 +24,7 @@ from .utils import ( is_cdna, ) from transformers.models.llama.modeling_llama import logger -from packaging.version import Version +from unsloth_zoo.utils import Version from unsloth_zoo.loss_utils import ( patch_loss_functions as _patch_loss_functions, diff --git a/unsloth/models/cohere.py b/unsloth/models/cohere.py index e9f56763d6..c33317ee02 100644 --- a/unsloth/models/cohere.py +++ b/unsloth/models/cohere.py @@ -15,7 +15,7 @@ from .llama import * from ._utils import __version__ from unsloth_zoo.hf_utils import dtype_from_config -from unsloth_zoo.utils import _get_dtype +from unsloth_zoo.utils import _get_dtype, Version from ..utils.packing import get_packed_info_from_kwargs from ..utils.attention_dispatch import ( AttentionConfig, @@ -35,8 +35,6 @@ try: repeat_kv, ) except: - from packaging.version import Version - transformers_version = Version(transformers_version) if not transformers_version >= Version("4.42"): raise ImportError( diff --git a/unsloth/models/gemma.py b/unsloth/models/gemma.py index 291d442673..1789a9cd92 100644 --- a/unsloth/models/gemma.py +++ b/unsloth/models/gemma.py @@ -14,7 +14,7 @@ from .llama import * from ._utils import __version__ -from unsloth_zoo.utils import _get_dtype +from unsloth_zoo.utils import _get_dtype, Version from unsloth_zoo.hf_utils import dtype_from_config from ..utils.packing import ( build_sdpa_packed_attention_mask, @@ -34,8 +34,6 @@ try: repeat_kv, ) except: - from packaging.version import Version - transformers_version = Version(transformers_version) if not transformers_version >= Version("4.38"): raise ImportError( diff --git a/unsloth/models/gemma2.py b/unsloth/models/gemma2.py index 4b2503b8a1..16d04955d3 100644 --- a/unsloth/models/gemma2.py +++ b/unsloth/models/gemma2.py @@ -14,7 +14,7 @@ from .llama import * from ._utils import __version__ -from unsloth_zoo.utils import _get_dtype +from unsloth_zoo.utils import _get_dtype, Version from unsloth_zoo.hf_utils import dtype_from_config from ..utils.packing import get_packed_info_from_kwargs from ..utils.attention_dispatch import ( @@ -41,8 +41,6 @@ try: repeat_kv, ) except: - from packaging.version import Version - transformers_version = Version(transformers_version) if not transformers_version >= Version("4.42"): raise ImportError( diff --git a/unsloth/models/granite.py b/unsloth/models/granite.py index f85f1b641f..aae746aed1 100644 --- a/unsloth/models/granite.py +++ b/unsloth/models/granite.py @@ -15,7 +15,7 @@ from .llama import * import os from ._utils import __version__ -from unsloth_zoo.utils import _get_dtype +from unsloth_zoo.utils import _get_dtype, Version from unsloth_zoo.hf_utils import dtype_from_config from ..utils.packing import get_packed_info_from_kwargs from ..utils.attention_dispatch import ( @@ -41,8 +41,6 @@ try: GraniteForCausalLM, ) except: - from packaging.version import Version - transformers_version = Version(transformers_version) if not transformers_version >= Version("4.45.0"): raise ImportError( diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py index 85332e1116..fe2a89d893 100644 --- a/unsloth/models/loader_utils.py +++ b/unsloth/models/loader_utils.py @@ -28,7 +28,6 @@ from .mapper import ( ) # https://github.com/huggingface/transformers/pull/26037 allows 4 bit loading! -from packaging.version import Version from transformers import __version__ as transformers_version from unsloth.models._utils import TorchAOConfig from unsloth_zoo.utils import Version diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py index 88aeeda8a1..35a15d03a6 100644 --- a/unsloth/models/rl.py +++ b/unsloth/models/rl.py @@ -43,10 +43,28 @@ torch_compile_options = { "triton.cudagraphs": False, } -from trl import __version__ as trl_version +# vLLM compatibility shim (TRL expects GuidedDecodingParams even if vLLM doesn't provide it) +try: + import vllm.sampling_params as _unsloth_vllm_sp + if not hasattr(_unsloth_vllm_sp, "GuidedDecodingParams"): + class GuidedDecodingParams: + def __init__(self, **kwargs): + self.kwargs = kwargs + _unsloth_vllm_sp.GuidedDecodingParams = GuidedDecodingParams +except Exception: + pass + +from trl import __version__ as trl_version_raw +from importlib.metadata import version as importlib_version from unsloth_zoo.utils import Version -trl_version = Version(trl_version) +try: + trl_version = Version(trl_version_raw) +except Exception: + try: + trl_version = Version(importlib_version("trl")) + except Exception: + trl_version = Version("0.0.0") def vLLMSamplingParams(**kwargs): @@ -220,7 +238,7 @@ RLTrainer_replacement = ''' import os from typing import * from dataclasses import dataclass, field -from packaging.version import Version +from unsloth_zoo.utils import Version import torch import numpy as np from contextlib import nullcontext @@ -242,12 +260,18 @@ def prepare_for_training_mode(f): @functools.wraps(f) def wrapper(self, *args, **kwargs): # Enable training mode + _was_training = None + if hasattr(self, 'model') and hasattr(self.model, "training"): + _was_training = self.model.training if hasattr(self, 'model') and hasattr(self.model, "for_training"): self.model.for_training() output = f(self, *args, **kwargs) - # Return inference mode + # Restore previous mode when possible if hasattr(self, 'model') and hasattr(self.model, "for_inference"): - self.model.for_inference() + if _was_training is False: + self.model.for_inference() + elif _was_training is True and hasattr(self.model, "for_training"): + self.model.for_training() # Reset gradient checkpointing buffers to free memory while staying ready for next run try: reset_unsloth_gradient_checkpointing_buffers() @@ -331,6 +355,27 @@ class Unsloth{RLTrainer_name}(_Unsloth{RLTrainer_name}): pass ''' +def _wrap_grpo_generate_and_score(trainer_cls): + if not hasattr(trainer_cls, "_generate_and_score_completions"): + return + original = trainer_cls._generate_and_score_completions + if getattr(original, "_unsloth_restore_training_wrapped", False): + return + + def wrapped(self, *args, **kwargs): + was_training = getattr(getattr(self, "model", None), "training", None) + try: + return original(self, *args, **kwargs) + finally: + if was_training is False and hasattr(self, "model") and hasattr(self.model, "for_inference"): + try: + self.model.for_inference() + except Exception: + pass + + wrapped._unsloth_restore_training_wrapped = True + trainer_cls._generate_and_score_completions = wrapped + def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"): # Patch for vLLM and Unsloth PEFT @@ -1059,6 +1104,16 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"): globals(), ) + if trainer_file == "grpo_trainer": + try: + _wrap_grpo_generate_and_score( + getattr(created_module, f"Unsloth{RLTrainer_name}") + ) + except Exception as e: + logger.info( + f"Unsloth: Could not wrap _generate_and_score_completions for {RLTrainer_name}: {e}" + ) + def patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, imports): init = inspect.getsource(RLTrainer.__init__) diff --git a/unsloth/trainer.py b/unsloth/trainer.py index 0d98cff305..858dcf2cd3 100644 --- a/unsloth/trainer.py +++ b/unsloth/trainer.py @@ -211,7 +211,7 @@ def _backwards_compatible_trainer(trainer_class, config_class): if "processing_class" in trainer_params and "tokenizer" in kwargs: kwargs["processing_class"] = kwargs.pop("tokenizer") - if ("args" in kwargs) and (Version(trl.__version__) >= Version("0.13.0.dev0")): + if ("args" in kwargs) and (Version(trl) >= Version("0.13.0.dev0")): training_args = kwargs.pop("args", None) # Get parameters that Trainer.__init__ actually expects @@ -412,7 +412,7 @@ def _patch_trl_trainer(): if hasattr(trl, "__UNSLOTH_BACKWARDS_COMPATIBLE__"): return - if Version(trl.__version__) <= Version("0.11.0"): + if Version(trl) <= Version("0.11.0"): return import trl.trainer From 1a9543fadd80e5df817690360ac80da000083064 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 5 Jan 2026 07:03:34 +0000 Subject: [PATCH 18/33] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- unsloth/models/rl.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py index 35a15d03a6..23cbbf0256 100644 --- a/unsloth/models/rl.py +++ b/unsloth/models/rl.py @@ -46,10 +46,13 @@ torch_compile_options = { # vLLM compatibility shim (TRL expects GuidedDecodingParams even if vLLM doesn't provide it) try: import vllm.sampling_params as _unsloth_vllm_sp + if not hasattr(_unsloth_vllm_sp, "GuidedDecodingParams"): + class GuidedDecodingParams: def __init__(self, **kwargs): self.kwargs = kwargs + _unsloth_vllm_sp.GuidedDecodingParams = GuidedDecodingParams except Exception: pass @@ -355,6 +358,7 @@ class Unsloth{RLTrainer_name}(_Unsloth{RLTrainer_name}): pass ''' + def _wrap_grpo_generate_and_score(trainer_cls): if not hasattr(trainer_cls, "_generate_and_score_completions"): return @@ -367,7 +371,11 @@ def _wrap_grpo_generate_and_score(trainer_cls): try: return original(self, *args, **kwargs) finally: - if was_training is False and hasattr(self, "model") and hasattr(self.model, "for_inference"): + if ( + was_training is False + and hasattr(self, "model") + and hasattr(self.model, "for_inference") + ): try: self.model.for_inference() except Exception: From aff2dc9061faba13bae73035ac47b780a21c60fb Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 5 Jan 2026 10:02:11 +0000 Subject: [PATCH 19/33] Add None check for vLLM tokenizer - Check _vllm_tok is not None before accessing attributes - Use getattr for safer chat_template access --- unsloth/models/rl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py index e1ecd6df2f..fd0c69bb0e 100644 --- a/unsloth/models/rl.py +++ b/unsloth/models/rl.py @@ -701,7 +701,7 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"): "if hasattr(self, 'llm') and self.llm is not None and hasattr(self.llm, 'get_tokenizer'):\n" " _vllm_tok = self.llm.get_tokenizer()\n" " _pc = getattr(self, 'processing_class', None) or getattr(self, 'tokenizer', None)\n" - " if _pc is not None and getattr(_pc, 'chat_template', None) is not None and _vllm_tok.chat_template is None:\n" + " if _vllm_tok is not None and _pc is not None and getattr(_pc, 'chat_template', None) is not None and getattr(_vllm_tok, 'chat_template', None) is None:\n" " _vllm_tok.chat_template = _pc.chat_template\n" "pass\n" ) From 6bf555a34c17820f3931f2e9ebfe8c9fb4fee229 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 5 Jan 2026 12:32:16 +0000 Subject: [PATCH 20/33] Remove unnecessary PDL module existence check Old vLLM versions without PDL modules don't need the fix. The patching code already handles missing modules gracefully. --- unsloth/import_fixes.py | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index 469674b29e..ef647ec65a 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -593,27 +593,6 @@ def fix_vllm_pdl_blackwell(): except Exception: return - # Helper to check if module spec exists - def _spec_exists(name): - try: - return importlib.util.find_spec(name) is not None - except (ModuleNotFoundError, ValueError): - return False - - # Check if vLLM has the PDL-related modules before doing internet check - has_utils = _spec_exists("vllm.lora.ops.triton_ops.utils") - has_expand_op = _spec_exists("vllm.lora.ops.triton_ops.lora_expand_op") - has_shrink_op = _spec_exists("vllm.lora.ops.triton_ops.lora_shrink_op") - - if not has_utils and not has_expand_op and not has_shrink_op: - # Old vLLM version without PDL support - just set env var to be safe - os.environ["TRITON_DISABLE_PDL"] = "1" - logger.info( - f"Unsloth: Set TRITON_DISABLE_PDL=1 for SM100 ({sm100_gpu_name}) - " - f"vLLM PDL modules not found" - ) - return - # Check if GitHub issue is closed (fix merged upstream) issue_closed = False try: From 9b6d536e0ee0ccc8eb2ddb9bb733044b182fd13e Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 5 Jan 2026 12:34:32 +0000 Subject: [PATCH 21/33] Keep PDL module check but remove unnecessary env var setting The check skips the GitHub API call for old vLLM versions. No need to set TRITON_DISABLE_PDL for versions without PDL support. --- unsloth/import_fixes.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index ef647ec65a..e8c4a2f665 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -593,6 +593,22 @@ def fix_vllm_pdl_blackwell(): except Exception: return + # Helper to check if module spec exists + def _spec_exists(name): + try: + return importlib.util.find_spec(name) is not None + except (ModuleNotFoundError, ValueError): + return False + + # Check if vLLM has the PDL-related modules before doing internet check + has_utils = _spec_exists("vllm.lora.ops.triton_ops.utils") + has_expand_op = _spec_exists("vllm.lora.ops.triton_ops.lora_expand_op") + has_shrink_op = _spec_exists("vllm.lora.ops.triton_ops.lora_shrink_op") + + if not has_utils and not has_expand_op and not has_shrink_op: + # Old vLLM version without PDL support - nothing to patch + return + # Check if GitHub issue is closed (fix merged upstream) issue_closed = False try: From 9ced3523aa73b9161ec87b2f9c2a62c3d0378b7a Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 5 Jan 2026 13:15:17 +0000 Subject: [PATCH 22/33] Replace GitHub API check with vLLM version check for PDL fix The GitHub issue check had issues: 1. Network latency on import 2. Issue being closed does not mean the fix is in the installed vLLM version Now skip the PDL workaround if vLLM version > 0.13.2, which is when the upstream fix is expected to be included. --- unsloth/import_fixes.py | 43 +++++++---------------------------------- 1 file changed, 7 insertions(+), 36 deletions(-) diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index e8c4a2f665..86a504b7b2 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -609,47 +609,18 @@ def fix_vllm_pdl_blackwell(): # Old vLLM version without PDL support - nothing to patch return - # Check if GitHub issue is closed (fix merged upstream) - issue_closed = False + # Check if vLLM version includes the fix (expected in versions > 0.13.2) try: - import socket - import urllib.request - import json as json_module - - # Quick internet connectivity check (0.5s timeout) - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.settimeout(0.5) - try: - sock.connect(("api.github.com", 443)) - has_internet = True - except (socket.timeout, OSError): - has_internet = False - finally: - sock.close() - - if has_internet: - api_url = "https://api.github.com/repos/vllm-project/vllm/issues/30872" - req = urllib.request.Request( - api_url, - headers = { - "User-Agent": "Unsloth-PDL-Fix", - "Accept": "application/vnd.github.v3+json", - }, + vllm_version = Version(importlib_version("vllm")) + if vllm_version > Version("0.13.2"): + logger.info( + f"Unsloth: SM100 ({sm100_gpu_name}) detected but vLLM {vllm_version} " + f"should include PDL fix - skipping workaround" ) - with urllib.request.urlopen(req, timeout = 3) as response: - data = json_module.loads(response.read().decode()) - issue_closed = data.get("state") == "closed" + return except Exception: - # If we can't check, assume issue is still open (apply fix to be safe) pass - if issue_closed: - logger.info( - f"Unsloth: SM100 ({sm100_gpu_name}) detected but PDL issue #30872 " - f"is closed - skipping PDL fix" - ) - return - # Apply the PDL fix os.environ["TRITON_DISABLE_PDL"] = "1" From cb42ce8dae17efa1f52c09104703da180e3eadd3 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 5 Jan 2026 13:19:37 +0000 Subject: [PATCH 23/33] Address review feedback: add constant and debug logging --- unsloth/import_fixes.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index 86a504b7b2..958173213d 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -609,17 +609,18 @@ def fix_vllm_pdl_blackwell(): # Old vLLM version without PDL support - nothing to patch return - # Check if vLLM version includes the fix (expected in versions > 0.13.2) + # Check if vLLM version includes the fix + VLLM_PDL_FIX_VERSION = "0.13.2" try: vllm_version = Version(importlib_version("vllm")) - if vllm_version > Version("0.13.2"): + if vllm_version > Version(VLLM_PDL_FIX_VERSION): logger.info( f"Unsloth: SM100 ({sm100_gpu_name}) detected but vLLM {vllm_version} " f"should include PDL fix - skipping workaround" ) return - except Exception: - pass + except Exception as e: + logger.debug(f"Unsloth: vLLM version check failed ({e}), applying PDL workaround.") # Apply the PDL fix os.environ["TRITON_DISABLE_PDL"] = "1" From c612bfe3a3472df436e951aee2930185d334b3c4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 5 Jan 2026 13:19:44 +0000 Subject: [PATCH 24/33] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- unsloth/import_fixes.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index 958173213d..1e05e462e9 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -620,7 +620,9 @@ def fix_vllm_pdl_blackwell(): ) return except Exception as e: - logger.debug(f"Unsloth: vLLM version check failed ({e}), applying PDL workaround.") + logger.debug( + f"Unsloth: vLLM version check failed ({e}), applying PDL workaround." + ) # Apply the PDL fix os.environ["TRITON_DISABLE_PDL"] = "1" From dc986cd7e2aa6ca24c100e3b8791956f966e5ca0 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Mon, 5 Jan 2026 13:29:58 +0000 Subject: [PATCH 25/33] Drop rl.py GRPO changes from this branch --- unsloth/models/rl.py | 86 ++++++++++---------------------------------- 1 file changed, 18 insertions(+), 68 deletions(-) diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py index 23cbbf0256..fd0c69bb0e 100644 --- a/unsloth/models/rl.py +++ b/unsloth/models/rl.py @@ -43,31 +43,10 @@ torch_compile_options = { "triton.cudagraphs": False, } -# vLLM compatibility shim (TRL expects GuidedDecodingParams even if vLLM doesn't provide it) -try: - import vllm.sampling_params as _unsloth_vllm_sp - - if not hasattr(_unsloth_vllm_sp, "GuidedDecodingParams"): - - class GuidedDecodingParams: - def __init__(self, **kwargs): - self.kwargs = kwargs - - _unsloth_vllm_sp.GuidedDecodingParams = GuidedDecodingParams -except Exception: - pass - -from trl import __version__ as trl_version_raw -from importlib.metadata import version as importlib_version +from trl import __version__ as trl_version from unsloth_zoo.utils import Version -try: - trl_version = Version(trl_version_raw) -except Exception: - try: - trl_version = Version(importlib_version("trl")) - except Exception: - trl_version = Version("0.0.0") +trl_version = Version(trl_version) def vLLMSamplingParams(**kwargs): @@ -241,7 +220,7 @@ RLTrainer_replacement = ''' import os from typing import * from dataclasses import dataclass, field -from unsloth_zoo.utils import Version +from packaging.version import Version import torch import numpy as np from contextlib import nullcontext @@ -263,18 +242,12 @@ def prepare_for_training_mode(f): @functools.wraps(f) def wrapper(self, *args, **kwargs): # Enable training mode - _was_training = None - if hasattr(self, 'model') and hasattr(self.model, "training"): - _was_training = self.model.training if hasattr(self, 'model') and hasattr(self.model, "for_training"): self.model.for_training() output = f(self, *args, **kwargs) - # Restore previous mode when possible + # Return inference mode if hasattr(self, 'model') and hasattr(self.model, "for_inference"): - if _was_training is False: - self.model.for_inference() - elif _was_training is True and hasattr(self.model, "for_training"): - self.model.for_training() + self.model.for_inference() # Reset gradient checkpointing buffers to free memory while staying ready for next run try: reset_unsloth_gradient_checkpointing_buffers() @@ -359,32 +332,6 @@ pass ''' -def _wrap_grpo_generate_and_score(trainer_cls): - if not hasattr(trainer_cls, "_generate_and_score_completions"): - return - original = trainer_cls._generate_and_score_completions - if getattr(original, "_unsloth_restore_training_wrapped", False): - return - - def wrapped(self, *args, **kwargs): - was_training = getattr(getattr(self, "model", None), "training", None) - try: - return original(self, *args, **kwargs) - finally: - if ( - was_training is False - and hasattr(self, "model") - and hasattr(self.model, "for_inference") - ): - try: - self.model.for_inference() - except Exception: - pass - - wrapped._unsloth_restore_training_wrapped = True - trainer_cls._generate_and_score_completions = wrapped - - def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"): # Patch for vLLM and Unsloth PEFT import trl @@ -747,6 +694,19 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"): ) RLTrainer_post += training_check + # Sync chat_template from processing_class to vLLM's tokenizer + # This fixes base models that have custom chat templates applied after loading + if "model" in call_args: + vllm_chat_template_sync = ( + "if hasattr(self, 'llm') and self.llm is not None and hasattr(self.llm, 'get_tokenizer'):\n" + " _vllm_tok = self.llm.get_tokenizer()\n" + " _pc = getattr(self, 'processing_class', None) or getattr(self, 'tokenizer', None)\n" + " if _vllm_tok is not None and _pc is not None and getattr(_pc, 'chat_template', None) is not None and getattr(_vllm_tok, 'chat_template', None) is None:\n" + " _vllm_tok.chat_template = _pc.chat_template\n" + "pass\n" + ) + RLTrainer_post += vllm_chat_template_sync + # Edit optional metrics other_metrics_processor = "" if trainer_file in RL_METRICS_CHANGES: @@ -1112,16 +1072,6 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"): globals(), ) - if trainer_file == "grpo_trainer": - try: - _wrap_grpo_generate_and_score( - getattr(created_module, f"Unsloth{RLTrainer_name}") - ) - except Exception as e: - logger.info( - f"Unsloth: Could not wrap _generate_and_score_completions for {RLTrainer_name}: {e}" - ) - def patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, imports): init = inspect.getsource(RLTrainer.__init__) From 7fd3a6c177fc242e94b16aee861bb541ae0c25c6 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Mon, 5 Jan 2026 13:39:03 +0000 Subject: [PATCH 26/33] Restore TRL version fallback in rl.py --- unsloth/models/rl.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py index fd0c69bb0e..11a5215c99 100644 --- a/unsloth/models/rl.py +++ b/unsloth/models/rl.py @@ -43,10 +43,28 @@ torch_compile_options = { "triton.cudagraphs": False, } -from trl import __version__ as trl_version +# vLLM compatibility shim (TRL expects GuidedDecodingParams even if vLLM doesn't provide it) +try: + import vllm.sampling_params as _unsloth_vllm_sp + if not hasattr(_unsloth_vllm_sp, "GuidedDecodingParams"): + class GuidedDecodingParams: + def __init__(self, **kwargs): + self.kwargs = kwargs + _unsloth_vllm_sp.GuidedDecodingParams = GuidedDecodingParams +except Exception: + pass + +from trl import __version__ as trl_version_raw +from importlib.metadata import version as importlib_version from unsloth_zoo.utils import Version -trl_version = Version(trl_version) +try: + trl_version = Version(trl_version_raw) +except Exception: + try: + trl_version = Version(importlib_version("trl")) + except Exception: + trl_version = Version("0.0.0") def vLLMSamplingParams(**kwargs): From 27e9a672a2ab2ff0d878c27e319646c3f03586a0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 5 Jan 2026 13:39:15 +0000 Subject: [PATCH 27/33] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- unsloth/models/rl.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py index 11a5215c99..2f6aef2709 100644 --- a/unsloth/models/rl.py +++ b/unsloth/models/rl.py @@ -46,10 +46,13 @@ torch_compile_options = { # vLLM compatibility shim (TRL expects GuidedDecodingParams even if vLLM doesn't provide it) try: import vllm.sampling_params as _unsloth_vllm_sp + if not hasattr(_unsloth_vllm_sp, "GuidedDecodingParams"): + class GuidedDecodingParams: def __init__(self, **kwargs): self.kwargs = kwargs + _unsloth_vllm_sp.GuidedDecodingParams = GuidedDecodingParams except Exception: pass From 506bcc48e54c43ca642f5fc0475ca9d5caf545d8 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Mon, 5 Jan 2026 13:50:48 +0000 Subject: [PATCH 28/33] Fix GRPO training state restoration --- unsloth/models/rl.py | 46 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py index 4ea36519d9..1327208c46 100644 --- a/unsloth/models/rl.py +++ b/unsloth/models/rl.py @@ -238,12 +238,18 @@ def prepare_for_training_mode(f): @functools.wraps(f) def wrapper(self, *args, **kwargs): # Enable training mode + _was_training = None + if hasattr(self, 'model') and hasattr(self.model, "training"): + _was_training = self.model.training if hasattr(self, 'model') and hasattr(self.model, "for_training"): self.model.for_training() output = f(self, *args, **kwargs) - # Return inference mode + # Restore previous mode when possible if hasattr(self, 'model') and hasattr(self.model, "for_inference"): - self.model.for_inference() + if _was_training is False: + self.model.for_inference() + elif _was_training is True and hasattr(self.model, "for_training"): + self.model.for_training() # Patch W&B to enable logging on future runs, otherwise it'll overwrite the first run try: import wandb @@ -323,6 +329,32 @@ pass ''' +def _wrap_grpo_generate_and_score(trainer_cls): + if not hasattr(trainer_cls, "_generate_and_score_completions"): + return + original = trainer_cls._generate_and_score_completions + if getattr(original, "_unsloth_restore_training_wrapped", False): + return + + def wrapped(self, *args, **kwargs): + was_training = getattr(getattr(self, "model", None), "training", None) + try: + return original(self, *args, **kwargs) + finally: + if ( + was_training is False + and hasattr(self, "model") + and hasattr(self.model, "for_inference") + ): + try: + self.model.for_inference() + except Exception: + pass + + wrapped._unsloth_restore_training_wrapped = True + trainer_cls._generate_and_score_completions = wrapped + + def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"): # Patch for vLLM and Unsloth PEFT import trl @@ -1046,6 +1078,16 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"): globals(), ) + if trainer_file == "grpo_trainer": + try: + _wrap_grpo_generate_and_score( + getattr(created_module, f"Unsloth{RLTrainer_name}") + ) + except Exception as e: + logger.info( + f"Unsloth: Could not wrap _generate_and_score_completions for {RLTrainer_name}: {e}" + ) + def patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, imports): init = inspect.getsource(RLTrainer.__init__) From 77e7f736419bf021c6e4502cea27939132a06bbd Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Mon, 5 Jan 2026 13:55:08 +0000 Subject: [PATCH 29/33] Revert rl_replacements GRPO edits --- unsloth/models/rl_replacements.py | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.py index f0f0386bd1..5e079335ae 100644 --- a/unsloth/models/rl_replacements.py +++ b/unsloth/models/rl_replacements.py @@ -259,7 +259,6 @@ def grpo_trainer__generate_and_score_completions(function_name, function): # The new multi-line string that will replace the line above replacement_lines = """ batch_size = self.args.per_device_train_batch_size if mode == "train" else self.args.per_device_eval_batch_size - _was_training = self.model.training try: # TRL 0.23.1 and below path if not has_images: @@ -389,20 +388,6 @@ def grpo_trainer__generate_and_score_completions(function_name, function): function = patched - match = re.search(r"^(\s*)return output", function, re.MULTILINE) - - if match: - indent = match.group(1) - new_code = ( - indent - + "if not _was_training:\n" - + indent - + " self.model.for_inference()\n" - + indent - + "return output" - ) - function = function.replace(f"{indent}return output", new_code) - return function @@ -876,13 +861,19 @@ def grpo_trainer_compute_loss(function_name, function): else torch.tensor(0.0, device = self.model.device) ) self._metrics[mode]["sampling/importance_sampling_ratio/min"].append( - nanmin(self.accelerator.gather(min_importance_sampling_ratio)).item() + self.accelerator.gather(min_importance_sampling_ratio) + .nan_to_num(nan = float("inf")) + .min() + .item() ) self._metrics[mode]["sampling/importance_sampling_ratio/mean"].append( self.accelerator.gather(mean_importance_sampling_ratio).nanmean().item() ) self._metrics[mode]["sampling/importance_sampling_ratio/max"].append( - nanmax(self.accelerator.gather(max_importance_sampling_ratio)).item() + self.accelerator.gather(max_importance_sampling_ratio) + .nan_to_num(nan = float("-inf")) + .max() + .item() ) return loss @@ -964,11 +955,15 @@ def openenv_vllm_reload_weights(): return if Version(importlib_version("trl")) < Version("0.26.0"): return + try: import trl.experimental.openenv.utils as openenv_utils import trl.experimental.openenv as openenv except ImportError as e: logger.info(f"Unsloth: Failed to import trl openenv: {e}") + logger.info( + "Unsloth: trl.experimental.openenv not available — skipping RL openenv patches." + ) return src = inspect.getsource(openenv_utils.generate_rollout_completions) From e7fe25ee43002eaf1cee7be2e2aa3fa6d3811f8a Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 5 Jan 2026 07:37:08 -0800 Subject: [PATCH 30/33] Versioning --- pyproject.toml | 4 ++-- unsloth/__init__.py | 2 +- unsloth/models/_utils.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e7b84f3c8e..7fa249e64c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,7 +60,7 @@ huggingfacenotorch = [ ] huggingface = [ "unsloth[huggingfacenotorch]", - "unsloth_zoo>=2026.1.1", + "unsloth_zoo>=2026.1.2", "torchvision", "unsloth[triton]", ] @@ -523,7 +523,7 @@ colab-ampere-torch220 = [ "flash-attn>=2.6.3 ; ('linux' in sys_platform)", ] colab-new = [ - "unsloth_zoo>=2026.1.1", + "unsloth_zoo>=2026.1.2", "packaging", "tyro", "transformers>=4.51.3,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1,!=4.57.0,<=4.57.3", diff --git a/unsloth/__init__.py b/unsloth/__init__.py index 86fb00fe0e..5b571cd456 100644 --- a/unsloth/__init__.py +++ b/unsloth/__init__.py @@ -79,7 +79,7 @@ from importlib.metadata import PackageNotFoundError # Check for unsloth_zoo try: unsloth_zoo_version = importlib_version("unsloth_zoo") - if Version(unsloth_zoo_version) < Version("2026.1.1"): + if Version(unsloth_zoo_version) < Version("2026.1.2"): print( "Unsloth: Please update Unsloth and Unsloth-Zoo to the latest version!\n" "Do this via `pip install --upgrade --force-reinstall --no-cache-dir --no-deps unsloth unsloth_zoo`" diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 5952d4af0c..b38c5860b3 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2026.1.1" +__version__ = "2026.1.2" __all__ = [ "SUPPORTS_BFLOAT16", From 9a5b824903e9071ba17c3fa6757185ddcd1287bf Mon Sep 17 00:00:00 2001 From: Datta Nimmaturi Date: Tue, 6 Jan 2026 09:53:20 +0000 Subject: [PATCH 31/33] Disable stats when modelscope is being used --- unsloth/models/_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index b38c5860b3..a2e1d78012 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -1197,7 +1197,8 @@ def get_statistics(local_files_only = False): # You can disable this by setting UNSLOTH_DISABLE_STATISTICS import os - if "UNSLOTH_DISABLE_STATISTICS" in os.environ: + global USE_MODELSCOPE + if "UNSLOTH_DISABLE_STATISTICS" in os.environ or USE_MODELSCOPE: return if local_files_only: return From 1d84ba52870c86f77820664bd598060d1db6bb39 Mon Sep 17 00:00:00 2001 From: Datta Nimmaturi Date: Tue, 6 Jan 2026 15:30:06 +0530 Subject: [PATCH 32/33] Check env var explicitly Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- unsloth/models/_utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index a2e1d78012..3cbb85ff4d 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -1197,8 +1197,7 @@ def get_statistics(local_files_only = False): # You can disable this by setting UNSLOTH_DISABLE_STATISTICS import os - global USE_MODELSCOPE - if "UNSLOTH_DISABLE_STATISTICS" in os.environ or USE_MODELSCOPE: + if "UNSLOTH_DISABLE_STATISTICS" in os.environ or os.environ.get("UNSLOTH_USE_MODELSCOPE", "0") == "1": return if local_files_only: return From 46d212c480b6d492b16b67493dab8ffe15a5b138 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 6 Jan 2026 10:00:16 +0000 Subject: [PATCH 33/33] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- unsloth/models/_utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 3cbb85ff4d..e6c4a12874 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -1197,7 +1197,10 @@ def get_statistics(local_files_only = False): # You can disable this by setting UNSLOTH_DISABLE_STATISTICS import os - if "UNSLOTH_DISABLE_STATISTICS" in os.environ or os.environ.get("UNSLOTH_USE_MODELSCOPE", "0") == "1": + if ( + "UNSLOTH_DISABLE_STATISTICS" in os.environ + or os.environ.get("UNSLOTH_USE_MODELSCOPE", "0") == "1" + ): return if local_files_only: return