ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

logits / temperature

Browse source
This commit is contained in:
Daniel Han 2025-06-22 04:54:18 -07:00
commit d9601bd14a
2 changed files with 16 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

11
unsloth/models/rl.py
View file

@ -568,6 +568,17 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"):
extra_args += check_num_generations
pass
# Check temperature must not be <= 0. Also stop if >= 10
if "temperature" in call_args:
check_temperature = \
"if temperature <= 0:\n"\
" raise MathError('Unsloth: Please set a positive non-zero temperature since your results will be wrong.')\n"\
"elif temperature >= 10:\n"\
" raise MathError('Unsloth: Please set a positive non-zero temperature less than 10, since sampling will be quite erratic.')\n"\
"\n"
extra_args += check_temperature
pass
# Edit config with anything extra
if trainer_file in RL_CONFIG_CHANGES:
process_extra_args = RL_CONFIG_CHANGES[trainer_file]

5
unsloth/models/rl_replacements.py
View file

@ -270,6 +270,8 @@ def grpo_trainer__get_per_token_logps(function_name, function):
# See https://github.com/huggingface/trl/issues/2770
# logits = logits[:, -logits_to_keep:]
# return logits
# See https://huggingface.co/blog/the_n_implementation_details_of_rlhf_with_ppo#policy-training-implementation-details
# logits = logits / self.temperature
# logps = selective_log_softmax(logits, input_ids)
# row_indices, col_indices = torch.where(logps < -20)
@ -358,6 +360,7 @@ def grpo_trainer_compute_loss(function_name, function):
epsilon_high = self.epsilon_high,
max_completion_length = self.args.max_completion_length,
delta = self.args.delta,
temperature = self.args.temperature,
)
else:
if hasattr(self.args, "loss_type"):
@ -374,6 +377,7 @@ def grpo_trainer_compute_loss(function_name, function):
epsilon_high = self.epsilon_high,
max_completion_length = self.args.max_completion_length,
delta = self.args.delta,
temperature = self.args.temperature,
)
else:
# to ensure backwards compatibility with trl 0.15.2 and maybe even 0.17
@ -385,6 +389,7 @@ def grpo_trainer_compute_loss(function_name, function):
advantages,
old_hidden_states,
n_chunks = self.args.unsloth_num_chunks,
temperature = self.args.temperature,
)
# Log the metrics