diff --git a/unsloth/__init__.py b/unsloth/__init__.py index 605df5a930..7932fe966d 100644 --- a/unsloth/__init__.py +++ b/unsloth/__init__.py @@ -29,6 +29,7 @@ from .import_fixes import ( fix_message_factory_issue, check_fbgemm_gpu_version, disable_broken_causal_conv1d, + disable_broken_vllm, configure_amdgpu_asic_id_table_path, torchvision_compatibility_check, fix_diffusers_warnings, @@ -38,6 +39,7 @@ from .import_fixes import ( # Configure libdrm ids table path early so ROCm can resolve AMD GPU names. configure_amdgpu_asic_id_table_path() disable_broken_causal_conv1d() +disable_broken_vllm() fix_message_factory_issue() check_fbgemm_gpu_version() torchvision_compatibility_check() @@ -45,6 +47,7 @@ fix_diffusers_warnings() fix_huggingface_hub() del configure_amdgpu_asic_id_table_path del disable_broken_causal_conv1d +del disable_broken_vllm del fix_message_factory_issue del check_fbgemm_gpu_version del torchvision_compatibility_check diff --git a/unsloth/dataprep/synthetic.py b/unsloth/dataprep/synthetic.py index 7735b1ecd2..612c531f47 100644 --- a/unsloth/dataprep/synthetic.py +++ b/unsloth/dataprep/synthetic.py @@ -27,11 +27,6 @@ import torch import gc import time import re -from unsloth_zoo.vllm_utils import ( - load_vllm, - patch_vllm, - delete_vllm, -) from unsloth_zoo.log import logger import numpy as np @@ -40,6 +35,16 @@ from .synthetic_configs import ( ) +def _load_vllm_utils(): + from unsloth_zoo.vllm_utils import ( + load_vllm, + patch_vllm, + delete_vllm, + ) + + return load_vllm, patch_vllm, delete_vllm + + def terminate_tree(proc: subprocess.Popen, timeout = 15): if proc is None or proc.poll() is not None: return @@ -182,6 +187,8 @@ class SyntheticDataKit: model_name, token = token, ) + load_vllm, patch_vllm, delete_vllm = _load_vllm_utils() + self._delete_vllm = delete_vllm patch_vllm(debug = False) engine_args = load_vllm( model_name = model_name, @@ -364,7 +371,8 @@ class SyntheticDataKit: gc.collect() # Delete vLLM module as well - delete_vllm(llm = None) + if hasattr(self, "_delete_vllm"): + self._delete_vllm(llm = None) def __enter__(self): return self diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index a86ad84779..0581cf7bdf 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -421,14 +421,18 @@ def fix_vllm_guided_decoding_params(): # trl still wants to use GuidedDecodingParams. This is a temporary patch till trl updates try: import vllm - except ImportError as e: + except (ImportError, OSError) as e: _maybe_raise_vllm_transformers_mismatch(e) + if disable_broken_vllm(e): + return raise try: from vllm.sampling_params import GuidedDecodingParams - except ImportError as e: + except (ImportError, OSError) as e: _maybe_raise_vllm_transformers_mismatch(e) + if disable_broken_vllm(e): + return if not hasattr(vllm, "sampling_params") or not hasattr( vllm.sampling_params, "StructuredOutputsParams" ): @@ -1028,7 +1032,7 @@ def fix_vllm_pdl_blackwell(): def _spec_exists(name): try: return importlib.util.find_spec(name) is not None - except (ModuleNotFoundError, ValueError): + except (ImportError, OSError, ModuleNotFoundError, ValueError): return False # Check if vLLM has the PDL-related modules before doing internet check @@ -1178,6 +1182,9 @@ def disable_torchcodec_if_broken(): CAUSAL_CONV1D_BROKEN = False _CAUSAL_CONV1D_PREFIX = "causal_conv1d" _CAUSAL_CONV1D_BLOCKER_SENTINEL = "_unsloth_causal_conv1d_blocker" +VLLM_BROKEN = False +_VLLM_PREFIX = "vllm" +_VLLM_BLOCKER_SENTINEL = "_unsloth_vllm_blocker" _ROCM_ENV_HINT_KEYS = ( "ROCM_PATH", "ROCM_HOME", @@ -1315,6 +1322,10 @@ def _is_causal_conv1d_name(module_name: str) -> bool: ) +def _is_vllm_name(module_name: str) -> bool: + return module_name == _VLLM_PREFIX or module_name.startswith(_VLLM_PREFIX + ".") + + def _resolve_module_name(module_name, package): if not isinstance(module_name, str): return module_name @@ -1344,6 +1355,27 @@ def _is_broken_causal_conv1d_error(error) -> bool: return False +def _is_broken_vllm_error(error) -> bool: + checked = set() + current = error + while current is not None and id(current) not in checked: + checked.add(id(current)) + message = str(current).lower() + if ( + ("vllm/_c" in message or "vllm._c" in message) + and ( + "undefined symbol" in message + or "cannot open shared object file" in message + or ".so:" in message + ) + ) or ("vllm" in message and "undefined symbol" in message): + return True + current = getattr(current, "__cause__", None) or getattr( + current, "__context__", None + ) + return False + + class _CausalConv1dImportBlockerLoader(importlib.abc.Loader): __slots__ = ("module_name",) @@ -1373,6 +1405,35 @@ class _CausalConv1dImportBlockerFinder(importlib.abc.MetaPathFinder): ) +class _VllmImportBlockerLoader(importlib.abc.Loader): + __slots__ = ("module_name",) + + def __init__(self, module_name): + self.module_name = module_name + + def create_module(self, spec): + return None + + def exec_module(self, module): + raise ModuleNotFoundError(f"No module named '{self.module_name}'") + + +class _VllmImportBlockerFinder(importlib.abc.MetaPathFinder): + __slots__ = (_VLLM_BLOCKER_SENTINEL,) + + def __init__(self): + setattr(self, _VLLM_BLOCKER_SENTINEL, True) + + def find_spec(self, fullname, path = None, target = None): + if not VLLM_BROKEN or not _is_vllm_name(fullname): + return None + return importlib.machinery.ModuleSpec( + name = fullname, + loader = _VllmImportBlockerLoader(fullname), + is_package = fullname == _VLLM_PREFIX, + ) + + def _patch_find_spec_for_causal_conv1d(): current_find_spec = importlib.util.find_spec if getattr(current_find_spec, "_unsloth_causal_conv1d_find_spec_patch", False): @@ -1390,6 +1451,23 @@ def _patch_find_spec_for_causal_conv1d(): importlib.util.find_spec = _blocked_find_spec +def _patch_find_spec_for_vllm(): + current_find_spec = importlib.util.find_spec + if getattr(current_find_spec, "_unsloth_vllm_find_spec_patch", False): + return + + def _blocked_find_spec(name, package = None): + resolved_name = _resolve_module_name(name, package) + if VLLM_BROKEN and isinstance(resolved_name, str): + if _is_vllm_name(resolved_name): + return None + return current_find_spec(name, package) + + _blocked_find_spec._unsloth_vllm_find_spec_patch = True + _blocked_find_spec._unsloth_original_find_spec = current_find_spec + importlib.util.find_spec = _blocked_find_spec + + def _install_causal_conv1d_blocker(): _patch_find_spec_for_causal_conv1d() for finder in sys.meta_path: @@ -1398,12 +1476,61 @@ def _install_causal_conv1d_blocker(): sys.meta_path.insert(0, _CausalConv1dImportBlockerFinder()) +def _install_vllm_blocker(): + _patch_find_spec_for_vllm() + for finder in sys.meta_path: + if getattr(finder, _VLLM_BLOCKER_SENTINEL, False): + return + sys.meta_path.insert(0, _VllmImportBlockerFinder()) + + def _clear_causal_conv1d_modules(): for module_name in list(sys.modules): if _is_causal_conv1d_name(module_name): sys.modules.pop(module_name, None) +def _clear_vllm_modules(): + for module_name in list(sys.modules): + if _is_vllm_name(module_name): + sys.modules.pop(module_name, None) + + +def disable_broken_vllm(error = None): + """Disable vLLM dynamically when its shared library is ABI-broken.""" + global VLLM_BROKEN + if VLLM_BROKEN: + _install_vllm_blocker() + return True + + failure = error + if failure is None: + try: + if importlib.util.find_spec("vllm") is None: + return False + except Exception: + return False + + try: + import vllm # noqa: F401 + + return False + except Exception as import_error: + failure = import_error + + if not _is_broken_vllm_error(failure): + return False + + VLLM_BROKEN = True + _clear_vllm_modules() + _install_vllm_blocker() + logger.warning( + "Unsloth: Detected broken vLLM binary extension; " + "disabling vLLM imports and continuing import." + ) + return True + + def _disable_transformers_causal_conv1d(): try: import transformers.utils.import_utils as tf_import_utils diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py index 01d221c725..f17ebc53e3 100644 --- a/unsloth/models/loader_utils.py +++ b/unsloth/models/loader_utils.py @@ -31,7 +31,6 @@ from .mapper import ( from transformers import __version__ as transformers_version from unsloth.models._utils import TorchAOConfig from unsloth_zoo.utils import Version -from unsloth_zoo.vllm_utils import _get_torchao_fp8_config import gc transformers_version = Version(transformers_version) @@ -49,6 +48,13 @@ BAD_MAPPINGS = { } +def _get_torchao_fp8_config(fp8_mode): + # Import lazily so an optional, broken vLLM install does not break plain `import unsloth`. + from unsloth_zoo.vllm_utils import _get_torchao_fp8_config as _impl + + return _impl(fp8_mode) + + def _get_env_int(keys): for key in keys: value = os.environ.get(key) diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py index 9e292a2849..00d8a3b7f2 100644 --- a/unsloth/models/vision.py +++ b/unsloth/models/vision.py @@ -126,11 +126,6 @@ _compile_config = CompileConfig( ) _compile_config.disable = True # Must set manually -from unsloth_zoo.vllm_utils import ( - convert_lora_modules, - return_lora_modules, -) - try: torch_compiler_set_stance = torch.compiler.set_stance except: