Update rl.py

unsloth/models/rl.py

@@ -537,6 +537,9 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"):
             "loss_type" : "bnpo",           # Default GRPO paper
             "beta" : 0.001,                 # Recommended as seen in verl
             "auto_find_batch_size" : False, # Cannot work on GRPO
+            # [TODO] See https://fengyao.notion.site/off-policy-rl
+            # https://github.com/huggingface/trl/pull/3867 (August 7th)
+            "vllm_importance_sampling_correction" : False,
         }
         for k, v in replacements.items():
             x = f"{k}( = [^,\n]{{1,}})?,\n"