diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py
index 581244e4d3..5651a7da41 100755
--- a/unsloth/models/rl.py
+++ b/unsloth/models/rl.py
@@ -273,8 +273,17 @@ def PatchRL(FastLanguageModel):
         with torch.no_grad():
             if has_labels or loss_without_labels:
                 with self.compute_loss_context_manager():
+                    try:
+                        num_items_in_batch = self._get_num_items_in_batch(
+                            [inputs], self.args.device
+                        )
+                    except (AttributeError, TypeError):
+                        num_items_in_batch = None
                     loss, outputs = self.compute_loss(
-                        model, inputs, return_outputs = True
+                        model,
+                        inputs,
+                        return_outputs = True,
+                        num_items_in_batch = num_items_in_batch,
                     )
                 loss = loss.mean().detach()