From 9e837e2dc385ff84c15baa9ddee92990aaa558f8 Mon Sep 17 00:00:00 2001
From: Daniel Han <danielhanchen@gmail.com>
Date: Wed, 10 Sep 2025 01:39:35 -0700
Subject: [PATCH] Update rl.py

---
 unsloth/models/rl.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py
index 14b75f6746..53f5eee66c 100644
--- a/unsloth/models/rl.py
+++ b/unsloth/models/rl.py
@@ -537,6 +537,9 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"):
             "loss_type" : "bnpo",           # Default GRPO paper
             "beta" : 0.001,                 # Recommended as seen in verl
             "auto_find_batch_size" : False, # Cannot work on GRPO
+            # [TODO] See https://fengyao.notion.site/off-policy-rl
+            # https://github.com/huggingface/trl/pull/3867 (August 7th)
+            "vllm_importance_sampling_correction" : False,
         }
         for k, v in replacements.items():
             x = f"{k}( = [^,\n]{{1,}})?,\n"