From fec06247c905cd07cdf5ef13337e00b04cdfc147 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 23 Feb 2026 01:06:53 -0800 Subject: [PATCH] Fix VLM processor load degradation and vLLM CUDA version detection (#4091) * Fix VLM processor load degradation and vLLM CUDA version detection vision.py - Fix VLM processor load for issue #4085: - Before loading the processor, scan local config files and strip the _Unsloth_Patched_ prefix. AutoProcessor.from_pretrained silently degrades to a text-only tokenizer instead of raising an exception when it encounters the unrecognized class name, so the existing get_auto_processor fallback never triggers. Sanitizing the configs before loading fixes backwards compat for old corrupted saves. - After loading, detect when AutoProcessor returned a text-only tokenizer for a VLM model (has no image_processor attribute) and trigger the manual fallback constructor. import_fixes.py - Fix vLLM CUDA version mismatch detection: - _is_broken_vllm_error now also matches CUDA shared library errors (libcudart, libcublas, libnvrtc) with "cannot open shared object file". Previously it only matched errors containing "vllm._c" in the message text, which missed cases where the error message was about the missing CUDA library itself (e.g. vllm built for CUDA 12 on a CUDA 13 system). - New _get_vllm_cuda_mismatch_message function extracts the CUDA version from the error, compares to the system CUDA version via torch.version.cuda, and returns a targeted install command using the correct GitHub releases wheel URL. - disable_broken_vllm uses the targeted message when a CUDA mismatch is detected, falling back to the existing generic message otherwise. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: Ubuntu Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- unsloth/import_fixes.py | 82 +++++++++++++++++++++++++++++++++++++--- unsloth/models/vision.py | 39 ++++++++++++++++++- 2 files changed, 113 insertions(+), 8 deletions(-) diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index b3200baa05..172ba36f63 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -1467,12 +1467,78 @@ def _is_broken_vllm_error(error) -> bool: ) ) or ("vllm" in message and "undefined symbol" in message): return True + # Also catch CUDA shared library mismatches during vllm import + # e.g. "libcudart.so.12: cannot open shared object file" + if ( + "libcudart" in message or "libcublas" in message or "libnvrtc" in message + ) and "cannot open shared object file" in message: + return True current = getattr(current, "__cause__", None) or getattr( current, "__context__", None ) return False +def _get_vllm_cuda_mismatch_message(error): + """If the error is a CUDA version mismatch, return a helpful install message.""" + import re as _re + + checked = set() + current = error + wanted_cuda = None + while current is not None and id(current) not in checked: + checked.add(id(current)) + message = str(current) + # Extract the CUDA version vllm was built for, e.g. "libcudart.so.12" + match = _re.search(r"libcudart\.so\.(\d+)", message) + if match: + wanted_cuda = match.group(1) + break + current = getattr(current, "__cause__", None) or getattr( + current, "__context__", None + ) + if wanted_cuda is None: + return None + + # Detect what CUDA version is actually available on the system + system_cuda_display = None # Human-readable, e.g. "13.0" + system_cuda_tag = None # For wheel URL, e.g. "130" + try: + import torch + + cuda_version = torch.version.cuda # e.g. "13.0" or "12.8" + if cuda_version: + system_cuda_display = cuda_version + system_cuda_tag = cuda_version.replace(".", "")[:3] # "130" or "128" + except Exception: + pass + + if system_cuda_tag is None or system_cuda_tag.startswith(wanted_cuda): + return None # Not a mismatch or can't determine + + try: + vllm_version = importlib_version("vllm").split("+")[0] + except Exception: + vllm_version = "VLLM_VERSION" + + cpu_arch = "x86_64" + try: + import platform + + cpu_arch = platform.machine() + except Exception: + pass + + return ( + f"Unsloth: vLLM was built for CUDA {wanted_cuda} but this system has " + f"CUDA {system_cuda_display}. Please reinstall vLLM with the correct CUDA version:\n" + f"\n" + f" uv pip install https://github.com/vllm-project/vllm/releases/download/" + f"v{vllm_version}/vllm-{vllm_version}+cu{system_cuda_tag}-cp38-abi3-" + f"manylinux_2_35_{cpu_arch}.whl" + ) + + class _CausalConv1dImportBlockerLoader(importlib.abc.Loader): __slots__ = ("module_name",) @@ -1621,12 +1687,16 @@ def disable_broken_vllm(error = None): VLLM_BROKEN = True _clear_vllm_modules() _install_vllm_blocker() - logger.warning( - "Unsloth: Detected broken vLLM binary extension; " - "disabling vLLM imports and continuing import.\n" - "Please reinstall via `uv pip install unsloth vllm torchvision torchaudio " - "--torch-backend=auto`." - ) + cuda_msg = _get_vllm_cuda_mismatch_message(failure) + if cuda_msg: + logger.warning(cuda_msg) + else: + logger.warning( + "Unsloth: Detected broken vLLM binary extension; " + "disabling vLLM imports and continuing import.\n" + "Please reinstall via `uv pip install unsloth vllm torchvision torchaudio " + "--torch-backend=auto`." + ) return True diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py index 00d8a3b7f2..2d6c7ac93f 100644 --- a/unsloth/models/vision.py +++ b/unsloth/models/vision.py @@ -916,6 +916,32 @@ class FastBaseModel: # Counteract saved tokenizers tokenizer_name = model_name if tokenizer_name is None else tokenizer_name + + # Fix _Unsloth_Patched_ prefix in local config files from old saves (issue #4085) + if os.path.isdir(tokenizer_name): + import json as _json + + for _cfg_name in ( + "processor_config.json", + "preprocessor_config.json", + "tokenizer_config.json", + ): + _cfg_path = os.path.join(tokenizer_name, _cfg_name) + if os.path.exists(_cfg_path): + try: + with open(_cfg_path, "r", encoding = "utf-8") as _f: + _cfg = _json.load(_f) + if _cfg.get("processor_class", "").startswith( + "_Unsloth_Patched_" + ): + _cfg["processor_class"] = _cfg["processor_class"][ + len("_Unsloth_Patched_") : + ] + with open(_cfg_path, "w", encoding = "utf-8") as _f: + _json.dump(_cfg, _f, indent = 2, ensure_ascii = False) + except Exception: + pass + if (whisper_language and whisper_task) or auto_model.__name__.endswith( "ForConditionalGeneration" ): @@ -947,14 +973,23 @@ class FastBaseModel: ) # If processor loading failed (e.g., tokenizer class not found), + # or if AutoProcessor silently degraded to a text-only tokenizer + # instead of returning a full VLM processor (issue #4085), # try constructing the processor manually from separate components. - if tokenizer is None and is_vlm: - tokenizer = _construct_vlm_processor_fallback( + _processor_is_degraded = ( + is_vlm + and tokenizer is not None + and not hasattr(tokenizer, "image_processor") + ) + if (tokenizer is None or _processor_is_degraded) and is_vlm: + _fallback = _construct_vlm_processor_fallback( tokenizer_name, model_type_arch, token, trust_remote_code, ) + if _fallback is not None: + tokenizer = _fallback if tokenizer is None: import sys