diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.py index 7462d55944..4288f53e6d 100644 --- a/unsloth/models/rl_replacements.py +++ b/unsloth/models/rl_replacements.py @@ -207,9 +207,12 @@ def grpo_trainer__get_per_token_logps(function_name, function): if function_name != "_get_per_token_logps": return function def _get_per_token_logps(self, model, input_ids, attention_mask, logits_to_keep): - return None # Unsloth efficient GRPO + if os.environ.get('UNSLOTH_USE_NEW_MODEL', '0') == '1': + return None # Unsloth efficient GRPO + # Otherwise, calculate normally: if not hasattr(self, '_autocast_dtype'): self._autocast_dtype = torch.float16 if os.environ.get('ACCELERATE_MIXED_PRECISION', 'fp16') == 'fp16' else torch.bfloat16 + if os.environ.get('UNSLOTH_FORCE_FLOAT32', '0') == '1': self._autocast_dtype = torch.float32 with torch.amp.autocast(device_type = 'cuda', dtype = self._autocast_dtype): # We add 1 to `logits_to_keep` because the last logits of the sequence is later excluded logits = model(input_ids=input_ids, attention_mask=attention_mask, logits_to_keep=logits_to_keep + 1).logits @@ -266,8 +269,8 @@ def grpo_trainer_compute_loss(function_name, function): # per_token_loss = -(per_token_loss - self.beta * per_token_kl) # loss = ((per_token_loss * completion_mask).sum(dim=1) / completion_mask.sum(dim=1)).mean() input_ids = input_ids[:, -logits_to_keep:] - if False:#per_token_logps is not None: - loss, completion_length, mean_kl = grpo_compute_loss( + if per_token_logps is not None: + loss, completion_length, mean_kl = grpo_compute_loss_compiled( ref_per_token_logps, per_token_logps, input_ids, completion_mask, self.beta, advantages, ) else: