ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add GRPO metrics (#1718)

Browse source 
* Update llama.py

* Update llama.py

* Faster inference?

* Update llama.py

* Update llama.py

* Update utils.py

* Update llama.py

* Update llama.py

* Update utils.py

* Update utils.py

* Update utils.py

* Update utils.py

* Update utils.py

* Update utils.py

* Update utils.py

* Update utils.py

* Update utils.py

* Update utils.py

* Update utils.py

* Update utils.py

* Update utils.py

* Update mapper.py

* Fast Inference via vLLM

* Update llama.py

* Update llama.py

* Update utils.py

* Create rl.py

* PatchRL

* Update rl.py

* Update rl.py

* Update rl.py

* PatchRLStatistics

* Update rl.py

* Update rl.py

* Update rl.py

* Update utils.py

* Update utils.py

* Update rl.py

* Update rl.py

* Update rl.py

* Update rl.py

* Update rl.py

* Update rl.py

* Update rl.py

* Update rl.py

* Update rl.py

* Update rl.py

* Update rl.py

* Update rl.py

* Update rl.py

* Update rl.py

* Update rl.py

* RL metrics

* Update rl.py

* RL metrics

* Update __init__.py

* Update rl.py

* Update rl.py

* Update rl.py

* Update chat_templates.py

* Update mapper.py

* Fp8 cache

* Update llama.py

* Update llama.py

* Update rl.py

* Update rl.py

* Update rl.py

* Update rl.py

* Update rl.py

* Update rl.py

* Update rl.py

* Update rl.py

* Update rl.py

* Update rl.py

* Update rl.py

* Update rl.py

* Update rl.py

* Update rl.py

* Update rl.py

* Update rl.py

* Update rl.py

* Update __init__.py

* Update loader.py

* Update rl.py

* Update rl.py

* Update _utils.py

* Update tokenizer_utils.py

* Update tokenizer_utils.py

* Better TRL handling

* Update rl.py

* Update tokenizer_utils.py

* Auto patching

* Update tokenizer_utils.py

* Update tokenizer_utils.py

* Update tokenizer_utils.py

* Update rl.py

* Update tokenizer_utils.py

* Update rl.py

* Update tokenizer_utils.py

* Update tokenizer_utils.py

* Update tokenizer_utils.py

* Update tokenizer_utils.py

* Update tokenizer_utils.py

* Update tokenizer_utils.py

* Update tokenizer_utils.py

* Update tokenizer_utils.py

* Update rl.py

* Update rl.py

* Update rl.py

* Update rl.py

* Update rl.py

* Update rl.py

* Update rl.py

* Update rl.py

* Update rl.py

* Update tokenizer_utils.py

* Update rl.py

* Update rl.py

* Update rl.py

* max seq length

* Update rl.py

* Update rl.py

* Patching

* Update rl.py

* Update rl.py

* Update rl.py

* Update rl.py

* Update rl.py

* NEFTune

* Update rl.py

* Update rl.py

* Update rl.py

* Update rl.py

* Update rl.py

* Update rl.py

* Update rl.py

* Extra replacements

* Update rl_replacements.py

* Update rl.py

* extra RL replacements

* Update rl_replacements.py

* Update rl_replacements.py

* Update rl_replacements.py

* Update rl_replacements.py

* Update rl_replacements.py

* Update rl_replacements.py

* Update llama.py

* Update rl_replacements.py

* Update _utils.py

* Update loader_utils.py

* Update rl.py

* Update rl_replacements.py

* Update rl_replacements.py

* Update rl.py

* Update llama.py

* Update llama.py

* Update llama.py

* Update llama.py

* autocast

* Update rl_replacements.py

* Update llama.py

* Update rl_replacements.py

* Update rl_replacements.py

* Update rl_replacements.py

* Update rl_replacements.py

* Update llama.py

* Update rl_replacements.py

* Update llama.py

* Update llama.py

* Update llama.py

* Update llama.py

* Update llama.py

* Update rl_replacements.py

* Update llama.py

* Update llama.py

* Update llama.py

* Update llama.py

* Update pyproject.toml

* Update llama.py

* Update llama.py

* Update llama.py

* Update llama.py

* Update llama.py

* Update llama.py

* Update llama.py

* Update rl_replacements.py

* Update rl_replacements.py

* Update rl_replacements.py

* Update rl_replacements.py

* Update llama.py

* Update rl_replacements.py

* Update rl_replacements.py

* Update rl_replacements.py

* Update rl_replacements.py

* Update rl_replacements.py

* Update rl_replacements.py

* Update rl_replacements.py

* Update rl_replacements.py

* Update llama.py

* Update _utils.py

* Update llama.py

* Update _utils.py

* Update rl_replacements.py

* Update rl.py

* Update rl.py

* Update rl.py

* Update rl.py

* Update rl.py

* Update llama.py

* Update llama.py

* Update llama.py

* Update llama.py

* Update rl_replacements.py

* Update llama.py

* Update llama.py

* Update llama.py

* Update llama.py

* GRPO optimized

* Update rl.py

* Update rl_replacements.py

* Update rl_replacements.py

* Update rl.py

* Update rl.py

* Update rl.py

* Update rl.py

* Update rl_replacements.py

* Update rl_replacements.py

* Update rl_replacements.py

* Selective Log softmax

* Fix GRPO bsz

* Update rl.py

* Update rl_replacements.py

* Update rl_replacements.py

* Update rl_replacements.py

* Update rl_replacements.py

* Fix TRL

* Metrics GRPO

* Update rl_replacements.py

* Update rl_replacements.py
This commit is contained in:
Daniel Han 2025-02-15 02:24:01 -08:00 • committed by GitHub
commit dc23ef819d
3 changed files with 36 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

2
unsloth/models/_utils.py
View file

@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
__version__ = "2025.2.10"
__version__ = "2025.2.11"
__all__ = [
"SUPPORTS_BFLOAT16",

13
unsloth/models/rl.py
View file

@ -30,6 +30,7 @@ from .rl_replacements import (
RL_FUNCTIONS,
RL_PRE_ITEMS,
RL_CONFIG_CHANGES,
RL_METRICS_CHANGES,
)
selective_log_softmax = RL_REPLACEMENTS["selective_log_softmax"]
@ -310,10 +311,20 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"):
RLTrainer_post += neftune_check
pass
# Edit optional metrics
other_metrics_processor = ""
if trainer_file in RL_METRICS_CHANGES:
process_extra_args = RL_METRICS_CHANGES[trainer_file]
for process_extra_arg in process_extra_args:
other_metrics_processor += process_extra_arg(call_args, extra_args)
pass
# Add statistics as well!
extra_args += \
"other_metrics = []\n"\
f"{other_metrics_processor}\n"\
"from unsloth_zoo.logging_utils import PatchRLStatistics\n"\
f"PatchRLStatistics('{trainer_file}')\n"
f"PatchRLStatistics('{trainer_file}', other_metrics)\n"
# Patch optional args
if trainer_file in RL_EXTRA_ARGS:

27
unsloth/models/rl_replacements.py
View file

@ -17,6 +17,7 @@ __all__ = [
"RL_FUNCTIONS",
"RL_PRE_ITEMS",
"RL_CONFIG_CHANGES",
"RL_METRICS_CHANGES",
]
import re
@ -24,10 +25,11 @@ import torch
import inspect
from collections import defaultdict
from unsloth_zoo.rl_replacements import RL_REPLACEMENTS
RL_EXTRA_ARGS = defaultdict(list)
RL_FUNCTIONS = defaultdict(list)
RL_PRE_ITEMS = defaultdict(list)
RL_CONFIG_CHANGES = defaultdict(list)
RL_EXTRA_ARGS = defaultdict(list)
RL_FUNCTIONS = defaultdict(list)
RL_PRE_ITEMS = defaultdict(list)
RL_CONFIG_CHANGES = defaultdict(list)
RL_METRICS_CHANGES = defaultdict(list)
torch_compile_options = {
"epilogue_fusion" : True,
@ -260,3 +262,20 @@ def grpo_trainer_fix_batch_size(RLTrainer_source, RLConfig_source):
return check_batch_size
pass
RL_CONFIG_CHANGES["grpo_trainer"].append(grpo_trainer_fix_batch_size)
# Add other reward function names
def grpo_trainer_metrics(RLTrainer_source, RLConfig_source):
if "reward_funcs" not in RLTrainer_source: return ""
log_metrics = \
"if not isinstance(reward_funcs, list): _reward_funcs = [reward_funcs]\n"\
"else: _reward_funcs = reward_funcs\n"\
"for reward_func in _reward_funcs:\n"\
" try:\n"\
" reward_func_name = reward_func.__name__\n"\
" other_metrics.append(f'rewards/{reward_func_name}')\n"\
" except: pass\n"
return log_metrics
pass
RL_METRICS_CHANGES["grpo_trainer"].append(grpo_trainer_metrics)