Fix VLM processor load degradation and vLLM CUDA version detection (#4091)
* Fix VLM processor load degradation and vLLM CUDA version detection vision.py - Fix VLM processor load for issue #4085: - Before loading the processor, scan local config files and strip the _Unsloth_Patched_ prefix. AutoProcessor.from_pretrained silently degrades to a text-only tokenizer instead of raising an exception when it encounters the unrecognized class name, so the existing get_auto_processor fallback never triggers. Sanitizing the configs before loading fixes backwards compat for old corrupted saves. - After loading, detect when AutoProcessor returned a text-only tokenizer for a VLM model (has no image_processor attribute) and trigger the manual fallback constructor. import_fixes.py - Fix vLLM CUDA version mismatch detection: - _is_broken_vllm_error now also matches CUDA shared library errors (libcudart, libcublas, libnvrtc) with "cannot open shared object file". Previously it only matched errors containing "vllm._c" in the message text, which missed cases where the error message was about the missing CUDA library itself (e.g. vllm built for CUDA 12 on a CUDA 13 system). - New _get_vllm_cuda_mismatch_message function extracts the CUDA version from the error, compares to the system CUDA version via torch.version.cuda, and returns a targeted install command using the correct GitHub releases wheel URL. - disable_broken_vllm uses the targeted message when a CUDA mismatch is detected, falling back to the existing generic message otherwise. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: Ubuntu <ubuntu@ip-172-31-16-253.us-east-2.compute.internal> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
This commit is contained in:
Daniel Han
GitHub
parent
3bddfed117
commit
fec06247c9
2 changed files with 113 additions and 8 deletions
|
|
@ -1467,12 +1467,78 @@ def _is_broken_vllm_error(error) -> bool:
|
|||
)
|
||||
) or ("vllm" in message and "undefined symbol" in message):
|
||||
return True
|
||||
# Also catch CUDA shared library mismatches during vllm import
|
||||
# e.g. "libcudart.so.12: cannot open shared object file"
|
||||
if (
|
||||
"libcudart" in message or "libcublas" in message or "libnvrtc" in message
|
||||
) and "cannot open shared object file" in message:
|
||||
return True
|
||||
current = getattr(current, "__cause__", None) or getattr(
|
||||
current, "__context__", None
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
def _get_vllm_cuda_mismatch_message(error):
|
||||
"""If the error is a CUDA version mismatch, return a helpful install message."""
|
||||
import re as _re
|
||||
|
||||
checked = set()
|
||||
current = error
|
||||
wanted_cuda = None
|
||||
while current is not None and id(current) not in checked:
|
||||
checked.add(id(current))
|
||||
message = str(current)
|
||||
# Extract the CUDA version vllm was built for, e.g. "libcudart.so.12"
|
||||
match = _re.search(r"libcudart\.so\.(\d+)", message)
|
||||
if match:
|
||||
wanted_cuda = match.group(1)
|
||||
break
|
||||
current = getattr(current, "__cause__", None) or getattr(
|
||||
current, "__context__", None
|
||||
)
|
||||
if wanted_cuda is None:
|
||||
return None
|
||||
|
||||
# Detect what CUDA version is actually available on the system
|
||||
system_cuda_display = None # Human-readable, e.g. "13.0"
|
||||
system_cuda_tag = None # For wheel URL, e.g. "130"
|
||||
try:
|
||||
import torch
|
||||
|
||||
cuda_version = torch.version.cuda # e.g. "13.0" or "12.8"
|
||||
if cuda_version:
|
||||
system_cuda_display = cuda_version
|
||||
system_cuda_tag = cuda_version.replace(".", "")[:3] # "130" or "128"
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if system_cuda_tag is None or system_cuda_tag.startswith(wanted_cuda):
|
||||
return None # Not a mismatch or can't determine
|
||||
|
||||
try:
|
||||
vllm_version = importlib_version("vllm").split("+")[0]
|
||||
except Exception:
|
||||
vllm_version = "VLLM_VERSION"
|
||||
|
||||
cpu_arch = "x86_64"
|
||||
try:
|
||||
import platform
|
||||
|
||||
cpu_arch = platform.machine()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return (
|
||||
f"Unsloth: vLLM was built for CUDA {wanted_cuda} but this system has "
|
||||
f"CUDA {system_cuda_display}. Please reinstall vLLM with the correct CUDA version:\n"
|
||||
f"\n"
|
||||
f" uv pip install https://github.com/vllm-project/vllm/releases/download/"
|
||||
f"v{vllm_version}/vllm-{vllm_version}+cu{system_cuda_tag}-cp38-abi3-"
|
||||
f"manylinux_2_35_{cpu_arch}.whl"
|
||||
)
|
||||
|
||||
|
||||
class _CausalConv1dImportBlockerLoader(importlib.abc.Loader):
|
||||
__slots__ = ("module_name",)
|
||||
|
||||
|
|
@ -1621,12 +1687,16 @@ def disable_broken_vllm(error = None):
|
|||
VLLM_BROKEN = True
|
||||
_clear_vllm_modules()
|
||||
_install_vllm_blocker()
|
||||
logger.warning(
|
||||
"Unsloth: Detected broken vLLM binary extension; "
|
||||
"disabling vLLM imports and continuing import.\n"
|
||||
"Please reinstall via `uv pip install unsloth vllm torchvision torchaudio "
|
||||
"--torch-backend=auto`."
|
||||
)
|
||||
cuda_msg = _get_vllm_cuda_mismatch_message(failure)
|
||||
if cuda_msg:
|
||||
logger.warning(cuda_msg)
|
||||
else:
|
||||
logger.warning(
|
||||
"Unsloth: Detected broken vLLM binary extension; "
|
||||
"disabling vLLM imports and continuing import.\n"
|
||||
"Please reinstall via `uv pip install unsloth vllm torchvision torchaudio "
|
||||
"--torch-backend=auto`."
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -916,6 +916,32 @@ class FastBaseModel:
|
|||
|
||||
# Counteract saved tokenizers
|
||||
tokenizer_name = model_name if tokenizer_name is None else tokenizer_name
|
||||
|
||||
# Fix _Unsloth_Patched_ prefix in local config files from old saves (issue #4085)
|
||||
if os.path.isdir(tokenizer_name):
|
||||
import json as _json
|
||||
|
||||
for _cfg_name in (
|
||||
"processor_config.json",
|
||||
"preprocessor_config.json",
|
||||
"tokenizer_config.json",
|
||||
):
|
||||
_cfg_path = os.path.join(tokenizer_name, _cfg_name)
|
||||
if os.path.exists(_cfg_path):
|
||||
try:
|
||||
with open(_cfg_path, "r", encoding = "utf-8") as _f:
|
||||
_cfg = _json.load(_f)
|
||||
if _cfg.get("processor_class", "").startswith(
|
||||
"_Unsloth_Patched_"
|
||||
):
|
||||
_cfg["processor_class"] = _cfg["processor_class"][
|
||||
len("_Unsloth_Patched_") :
|
||||
]
|
||||
with open(_cfg_path, "w", encoding = "utf-8") as _f:
|
||||
_json.dump(_cfg, _f, indent = 2, ensure_ascii = False)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if (whisper_language and whisper_task) or auto_model.__name__.endswith(
|
||||
"ForConditionalGeneration"
|
||||
):
|
||||
|
|
@ -947,14 +973,23 @@ class FastBaseModel:
|
|||
)
|
||||
|
||||
# If processor loading failed (e.g., tokenizer class not found),
|
||||
# or if AutoProcessor silently degraded to a text-only tokenizer
|
||||
# instead of returning a full VLM processor (issue #4085),
|
||||
# try constructing the processor manually from separate components.
|
||||
if tokenizer is None and is_vlm:
|
||||
tokenizer = _construct_vlm_processor_fallback(
|
||||
_processor_is_degraded = (
|
||||
is_vlm
|
||||
and tokenizer is not None
|
||||
and not hasattr(tokenizer, "image_processor")
|
||||
)
|
||||
if (tokenizer is None or _processor_is_degraded) and is_vlm:
|
||||
_fallback = _construct_vlm_processor_fallback(
|
||||
tokenizer_name,
|
||||
model_type_arch,
|
||||
token,
|
||||
trust_remote_code,
|
||||
)
|
||||
if _fallback is not None:
|
||||
tokenizer = _fallback
|
||||
if tokenizer is None:
|
||||
import sys
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue