From 3e1ceff3078ab25b84217e25cc91474cd608e3a4 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 5 Jan 2026 05:03:56 +0000 Subject: [PATCH 1/4] Sync chat_template from tokenizer to vLLM When using base models with custom chat templates applied after loading, vLLM's internal tokenizer may not have the chat_template set. This causes issues during RL training with vLLM inference. This fix syncs the chat_template from the processing_class (the tokenizer you loaded and configured) to vLLM's internal tokenizer during trainer initialization, but only if vLLM's tokenizer does not already have one set. --- unsloth/models/rl.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py index 88aeeda8a1..20dafaaaa4 100644 --- a/unsloth/models/rl.py +++ b/unsloth/models/rl.py @@ -694,6 +694,20 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"): ) RLTrainer_post += training_check + # Sync chat_template from processing_class to vLLM's tokenizer + # This fixes base models that have custom chat templates applied after loading + if "model" in call_args: + vllm_chat_template_sync = ( + "if hasattr(self, 'llm') and self.llm is not None and hasattr(self.llm, 'get_tokenizer'):\n" + " _vllm_tok = self.llm.get_tokenizer()\n" + " _pc = getattr(self, 'processing_class', None)\n" + " if _pc is not None and getattr(_pc, 'chat_template', None) is not None:\n" + " if _vllm_tok.chat_template is None:\n" + " _vllm_tok.chat_template = _pc.chat_template\n" + "pass\n" + ) + RLTrainer_post += vllm_chat_template_sync + # Edit optional metrics other_metrics_processor = "" if trainer_file in RL_METRICS_CHANGES: From abaee73d745cbcbbbef1b5279f6b1c85a9ec693f Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 5 Jan 2026 05:10:24 +0000 Subject: [PATCH 2/4] Add tokenizer fallback for chat_template sync --- unsloth/models/rl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py index 20dafaaaa4..b75ae383db 100644 --- a/unsloth/models/rl.py +++ b/unsloth/models/rl.py @@ -700,7 +700,7 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"): vllm_chat_template_sync = ( "if hasattr(self, 'llm') and self.llm is not None and hasattr(self.llm, 'get_tokenizer'):\n" " _vllm_tok = self.llm.get_tokenizer()\n" - " _pc = getattr(self, 'processing_class', None)\n" + " _pc = getattr(self, 'processing_class', None) or getattr(self, 'tokenizer', None)\n" " if _pc is not None and getattr(_pc, 'chat_template', None) is not None:\n" " if _vllm_tok.chat_template is None:\n" " _vllm_tok.chat_template = _pc.chat_template\n" From a5200296362bb66f3a8b953a8b40375b8f4eec74 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 5 Jan 2026 05:25:53 +0000 Subject: [PATCH 3/4] Combine nested if statements for clarity --- unsloth/models/rl.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py index b75ae383db..e1ecd6df2f 100644 --- a/unsloth/models/rl.py +++ b/unsloth/models/rl.py @@ -701,9 +701,8 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"): "if hasattr(self, 'llm') and self.llm is not None and hasattr(self.llm, 'get_tokenizer'):\n" " _vllm_tok = self.llm.get_tokenizer()\n" " _pc = getattr(self, 'processing_class', None) or getattr(self, 'tokenizer', None)\n" - " if _pc is not None and getattr(_pc, 'chat_template', None) is not None:\n" - " if _vllm_tok.chat_template is None:\n" - " _vllm_tok.chat_template = _pc.chat_template\n" + " if _pc is not None and getattr(_pc, 'chat_template', None) is not None and _vllm_tok.chat_template is None:\n" + " _vllm_tok.chat_template = _pc.chat_template\n" "pass\n" ) RLTrainer_post += vllm_chat_template_sync From 0a07009eb5d0cf470dd8d7a0fdcdcece171b6b43 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 5 Jan 2026 10:02:11 +0000 Subject: [PATCH 4/4] Add None check for vLLM tokenizer - Check _vllm_tok is not None before accessing attributes - Use getattr for safer chat_template access --- unsloth/models/rl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py index e1ecd6df2f..fd0c69bb0e 100644 --- a/unsloth/models/rl.py +++ b/unsloth/models/rl.py @@ -701,7 +701,7 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"): "if hasattr(self, 'llm') and self.llm is not None and hasattr(self.llm, 'get_tokenizer'):\n" " _vllm_tok = self.llm.get_tokenizer()\n" " _pc = getattr(self, 'processing_class', None) or getattr(self, 'tokenizer', None)\n" - " if _pc is not None and getattr(_pc, 'chat_template', None) is not None and _vllm_tok.chat_template is None:\n" + " if _vllm_tok is not None and _pc is not None and getattr(_pc, 'chat_template', None) is not None and getattr(_vllm_tok, 'chat_template', None) is None:\n" " _vllm_tok.chat_template = _pc.chat_template\n" "pass\n" )