diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py
index 7a90b81157..3b7b88b6c3 100644
--- a/unsloth/models/rl.py
+++ b/unsloth/models/rl.py
@@ -455,14 +455,14 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"):
         RLTrainer_arguments  = RLTrainer_arguments,
         RLTrainer_extra_args = RLTrainer_extra_args,
         RLTrainer_call_args  = RLTrainer_call_args,
-        RLTrainer_kwargs     = unsloth_extra_args,
+        RLTrainer_kwargs     = ",**kwargs"[1 if RLTrainer_call_args .endswith(",") else 0:],
 
         RLConfig_name        = RLConfig_name,
         __RLConfig_doc__     = __RLConfig_doc__,
         RLConfig_arguments   = RLConfig_arguments,
         RLConfig_extra_args  = RLConfig_extra_args,
         RLConfig_call_args   = RLConfig_call_args,
-        RLConfig_kwargs      = ",**kwargs"[1 if RLConfig_call_args .endswith(",") else 0:],
+        RLConfig_kwargs      = unsloth_extra_args,
 
         RLTrainer_extras     = RLTrainer_extras,
         RLTrainer_post       = RLTrainer_post,