ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

vllm sampling params fix

Browse source
This commit is contained in:
Datta Nimmaturi 2025-12-01 10:39:52 +00:00
commit 85704eb57b
3 changed files with 61 additions and 39 deletions
Download patch file Download diff file Expand all files Collapse all files

6
unsloth/__init__.py
View file

@ -109,7 +109,7 @@ from unsloth_zoo.device_type import (
# Fix other issues
from .import_fixes import (
fix_xformers_performance_issue,
fix_vllm_aimv2_issue,
patch_vllm_imports,
ignore_logger_messages,
patch_ipykernel_hf_xet,
patch_trackio,
@ -117,14 +117,14 @@ from .import_fixes import (
)
fix_xformers_performance_issue()
fix_vllm_aimv2_issue()
patch_vllm_imports()
ignore_logger_messages()
patch_ipykernel_hf_xet()
patch_trackio()
patch_datasets()
del fix_xformers_performance_issue
del fix_vllm_aimv2_issue
del patch_vllm_imports
del ignore_logger_messages
del patch_ipykernel_hf_xet
del patch_trackio

87
unsloth/import_fixes.py
View file

@ -114,45 +114,60 @@ def fix_xformers_performance_issue():
print(f"Unsloth: Failed patching Xformers with error = {str(e)}")
# ValueError: 'aimv2' is already used by a Transformers config, pick another name.
def fix_vllm_aimv2_issue():
def patch_vllm_imports():
if importlib.util.find_spec("vllm") is None:
return
vllm_version = importlib_version("vllm")
if Version(vllm_version) < Version("0.10.1"):
vllm_version = importlib.util.find_spec("vllm").origin
vllm_version = os.path.split(vllm_version)[0]
ovis_config = Path(vllm_version) / "transformers_utils" / "configs" / "ovis.py"
try:
if ovis_config.exists():
with open(ovis_config, "r+", encoding = "utf-8") as f:
text = f.read()
# See https://github.com/vllm-project/vllm-ascend/issues/2046
if 'AutoConfig.register("aimv2", AIMv2Config)' in text:
text = text.replace(
'AutoConfig.register("aimv2", AIMv2Config)',
"",
)
text = text.replace(
"""backbone_config.pop('model_type')
backbone_config = AutoConfig.for_model(model_type,
**backbone_config)""",
"""if model_type != "aimv2":
backbone_config.pop('model_type')
backbone_config = AutoConfig.for_model(model_type, **backbone_config)
else:
backbone_config = AIMv2Config(**backbone_config)""",
)
f.seek(0)
f.write(text)
f.truncate()
if UNSLOTH_ENABLE_LOGGING:
print(
"Unsloth: Patching vLLM to fix `'aimv2' is already used by a Transformers config, pick another name.`"
def fix_vllm_aimv2_issue():
# ValueError: 'aimv2' is already used by a Transformers config, pick another name.
vllm_version = importlib_version("vllm")
if Version(vllm_version) < Version("0.10.1"):
vllm_version = importlib.util.find_spec("vllm").origin
vllm_version = os.path.split(vllm_version)[0]
ovis_config = Path(vllm_version) / "transformers_utils" / "configs" / "ovis.py"
try:
if ovis_config.exists():
with open(ovis_config, "r+", encoding = "utf-8") as f:
text = f.read()
# See https://github.com/vllm-project/vllm-ascend/issues/2046
if 'AutoConfig.register("aimv2", AIMv2Config)' in text:
text = text.replace(
'AutoConfig.register("aimv2", AIMv2Config)',
"",
)
except Exception as e:
if UNSLOTH_ENABLE_LOGGING:
print(f"Unsloth: Failed patching vLLM with error = {str(e)}")
text = text.replace(
"""backbone_config.pop('model_type')
backbone_config = AutoConfig.for_model(model_type,
**backbone_config)""",
"""if model_type != "aimv2":
backbone_config.pop('model_type')
backbone_config = AutoConfig.for_model(model_type, **backbone_config)
else:
backbone_config = AIMv2Config(**backbone_config)""",
)
f.seek(0)
f.write(text)
f.truncate()
if UNSLOTH_ENABLE_LOGGING:
print(
"Unsloth: Patching vLLM to fix `'aimv2' is already used by a Transformers config, pick another name.`"
)
except Exception as e:
if UNSLOTH_ENABLE_LOGGING:
print(f"Unsloth: Failed patching vLLM with error = {str(e)}")
def fix_vllm_guided_decoding_params():
# GuidedDecodingParmas is renamed to StructuredOutputsParams in vLLM
# https://github.com/vllm-project/vllm/pull/22772/files
# trl still wants to use GuidedDecodingParams. This is a temporary patch till trl updates
import vllm
try:
from vllm.sampling_params import GuidedDecodingParams
except ImportError:
vllm.sampling_params.GuidedDecodingParams = vllm.sampling_params.StructuredOutputsParams
fix_vllm_aimv2_issue()
fix_vllm_guided_decoding_params()
def ignore_logger_messages():

7
unsloth/models/rl.py
View file

@ -329,6 +329,7 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"):
try:
trainer = eval(f"trl.trainer.{trainer_file}")
except Exception as error:
print(f"Unsloth: Could not import trl.trainer.{trainer_file}: {error}")
return
# Get SFTTrainer and SFTConfig names
@ -347,8 +348,10 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"):
and trainer_file.split("_")[0] in x.lower()
]
if len(name) != 1:
print(f"Unsloth: Could not find Trainer class in trl.trainer.{trainer_file}. Found: {name}")
return
if len(config) != 1:
print(f"Unsloth: Could not find Config class in trl.trainer.{trainer_file}. Found: {config}")
return
# Get SFTTrainer, SFTConfig
@ -357,16 +360,20 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"):
try:
RLTrainer = eval(f"trl.trainer.{trainer_file}.{RLTrainer_name}")
except:
print(f"Unsloth: Could not load {RLTrainer_name} from trl.trainer.{trainer_file}")
return
try:
RLConfig = eval(f"trl.trainer.{trainer_file}.{RLConfig_name}")
except:
print(f"Unsloth: Could not load {RLConfig_name} from trl.trainer.{trainer_file}")
return
# Check name
if RLTrainer.__name__.startswith("Unsloth"):
print(f"Unsloth: {RLTrainer.__name__} is already patched.")
return
if RLConfig.__name__.startswith("Unsloth"):
print(f"Unsloth: {RLConfig.__name__} is already patched.")
return
# Get old source