From 39bdd266ffdf47bc32fdb86c4bbc00403cd86a1f Mon Sep 17 00:00:00 2001
From: "google-labs-jules[bot]"
 <161369871+google-labs-jules[bot]@users.noreply.github.com>
Date: Sat, 6 Dec 2025 02:50:00 +0000
Subject: [PATCH] Remove reload_weights rpc call from grpo trainer

This commit removes the `reload_weights` RPC call from `_generate_single_turn` in `GRPOTrainer` via Unsloth's runtime patching mechanism. This call, introduced in recent `trl` versions, causes a `ValueError` during training when using Unsloth (specifically with LoRA) because it attempts to reload weights that are not fully initialized or managed by vLLM in the expected way.

The fix uses `re.sub` to robustly remove the offending line from the source code at runtime.

Fixes #3673
---
 unsloth/models/rl_replacements.py | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.py
index 38973fe30e..aa53ed21fa 100644
--- a/unsloth/models/rl_replacements.py
+++ b/unsloth/models/rl_replacements.py
@@ -216,6 +216,24 @@ def grpo_trainer__prepare_inputs(function_name, function):
 RL_FUNCTIONS["grpo_trainer"].append(grpo_trainer__prepare_inputs)
 
 
+# Remove reload_weights rpc call
+def grpo_trainer__generate_single_turn(function_name, function):
+    if function_name != "_generate_single_turn":
+        return function
+
+    # Remove reload_weights rpc call
+    function = re.sub(
+        r"^\s*self\.llm\.collective_rpc\([\"']reload_weights[\"']\)\s*\n",
+        "",
+        function,
+        flags = re.MULTILINE,
+    )
+    return function
+
+
+RL_FUNCTIONS["grpo_trainer"].append(grpo_trainer__generate_single_turn)
+
+
 # Fix incorrect special tokens handling and truncation in older TRL versions
 def grpo_trainer__generate_and_score_completions(function_name, function):
     if function_name != "_generate_and_score_completions":