From 9e3f3671d0062866c1835ecda273315c1a180337 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sun, 26 Jul 2026 14:59:27 +0000 Subject: [PATCH] docker: give llama.cpp its libcublas so GGUF stops running on the CPU The portable llama.cpp bundle loads libggml-cuda.so with dlopen, links it against libcublas, and does not ship libcublas. The CUDA runtime base image only carries libcudart, and the only libcublas in the image is torch's wheel copy under site-packages/nvidia/cublas/lib, which was not on the loader path. So the CUDA backend failed to load, and llama.cpp said nothing about it: `--list-devices` printed an empty list and every GGUF request ran on the CPU. Measured in the built image on a B200 with gemma-4-E2B-it UD-Q4_K_XL: 1.6 tok/s from llama-cli and 4.2 tok/s from llama-server. With the fix, the same image and model report `CUDA0: NVIDIA B200` and run at 229 tok/s and 193 tok/s. Studio's GGUF chat and the GGUF export path go through the same bundle, so both were affected. The venv loader config already existed for torchcodec, so cublas/lib joins it there rather than on LD_LIBRARY_PATH: ld.so.conf.d is consulted after DT_RUNPATH, which keeps llama.cpp resolving its own $ORIGIN libs first. cu13/lib comes along for the arm64 bundle's layout. A silent 140x slowdown deserves a build-time gate, so the layer after the fetch runs ldd over libggml-cuda.so, installs the cublas major the bundle actually asks for when it is missing, and fails the build on anything still unresolved. The amd64 bundle wants libcublas.so.12 and torch already provides it; the arm64 bundle is CUDA 13, and deriving the major from ldd keeps that leg honest without hardcoding either. libcuda.so.1 is exempt: nvidia-container-toolkit injects the driver stub at `docker run --gpus`, so it is never resolvable at build time. ldd needs no GPU, so the build stays host-independent. tests/python/test_docker_llama_cuda_backend.py pins the loader entry, the guard, the driver-stub exemption and the ordering. --- docker/Dockerfile | 39 +++++++- .../python/test_docker_llama_cuda_backend.py | 89 +++++++++++++++++++ 2 files changed, 127 insertions(+), 1 deletion(-) create mode 100644 tests/python/test_docker_llama_cuda_backend.py diff --git a/docker/Dockerfile b/docker/Dockerfile index 35083878c1..dc29131350 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -432,10 +432,13 @@ RUN set -eux; \ # Register the venv's torch + NVIDIA lib dirs with the loader so torchcodec can # dlopen them. ld.so.conf.d, NOT LD_LIBRARY_PATH: the cache is consulted after # DT_RUNPATH, so llama.cpp keeps resolving its own $ORIGIN libs first. +# cublas/lib and cu13/lib are here for llama.cpp's libggml-cuda.so, which links +# against libcublas but does not ship it (see the guard after the fetch below). RUN set -eux \ && SP=/opt/unsloth-venv/lib/python${PYTHON_VERSION}/site-packages \ && printf "%s\n" "$SP/torch/lib" "$SP/nvidia/cuda_nvrtc/lib" \ "$SP/nvidia/cuda_runtime/lib" "$SP/nvidia/npp/lib" \ + "$SP/nvidia/cublas/lib" "$SP/nvidia/cu13/lib" \ > /etc/ld.so.conf.d/zz-unsloth-venv.conf \ && ldconfig \ && { /opt/unsloth-venv/bin/python -c \ @@ -450,7 +453,7 @@ RUN set -eux \ # target (see fetch_llama_prebuilt.py): # * amd64 -> app--linux-x64-cuda12-portable.tar.gz (sm_70..sm_120) # arm64 -> app--linux-arm64-cuda13-portable.tar.gz (sm_90..sm_121) -# * portable bundles carry their own CUDA libs, so they also run CPU-only +# * portable bundles carry their own ggml backends, so they also run CPU-only # * sha256-verified against the release's llama-prebuilt-sha256.json # * converter + gguf-py from the SAME release's source tarball (mappings match) # /opt (not /root) so it survives `docker run --user`. Default "latest" resolves @@ -462,6 +465,40 @@ RUN /opt/unsloth-venv/bin/python /tmp/fetch_llama_prebuilt.py \ "${LLAMA_PREBUILT_TAG}" "${TARGETARCH:-amd64}" /opt/unsloth/llama.cpp \ && rm -f /tmp/fetch_llama_prebuilt.py \ && cat /opt/unsloth/llama.cpp/UNSLOTH_PREBUILT_INFO.json + +# libggml-cuda.so is loaded with dlopen (ggml_backend_dl), links against +# libcublas, and does not ship it; the CUDA runtime base only carries libcudart. +# A missing libcublas therefore makes the backend fail to load SILENTLY and +# llama.cpp runs on the CPU: measured 1.6 tok/s instead of 222 tok/s for +# gemma-4-E2B UD-Q4_K_XL on a B200, with `--list-devices` printing nothing. +# torch's wheels already ship libcublas for their own CUDA major (registered +# with the loader above); install the bundle's major when it differs. Then fail +# the build on any dependency that is still unresolved, so a silent CPU fallback +# can never ship again. libcuda.so.1 is exempt: that is the driver stub, injected +# by nvidia-container-toolkit at `docker run --gpus`, never present in the image. +# ldd needs no GPU, so this keeps the build host-independent. +RUN set -eux \ + && CUDA_SO=/opt/unsloth/llama.cpp/libggml-cuda.so \ + && if [ -f "$CUDA_SO" ]; then \ + want="$(ldd "$CUDA_SO" | sed -n 's/^[[:space:]]*\(libcublas\.so\.[0-9]*\)[[:space:]]*=> not found$/\1/p' | head -n1)"; \ + if [ -n "$want" ]; then \ + major="${want##*.}"; \ + echo ">> $want missing, installing nvidia-cublas-cu${major}"; \ + /opt/unsloth-venv/bin/uv pip install --python /opt/unsloth-venv/bin/python \ + "nvidia-cublas-cu${major}"; \ + ldconfig; \ + fi; \ + missing="$(ldd "$CUDA_SO" | grep 'not found' | grep -v 'libcuda\.so\.1 ' || true)"; \ + if [ -n "$missing" ]; then \ + echo "ERROR: llama.cpp CUDA backend has unresolved libraries:"; \ + echo "$missing"; \ + echo "GGUF inference would silently fall back to the CPU."; \ + exit 1; \ + fi; \ + echo "OK: llama.cpp CUDA backend dependencies all resolve"; \ + else \ + echo ">> no libggml-cuda.so in this bundle (CPU-only build)"; \ + fi ENV UNSLOTH_LLAMA_CPP_PATH=/opt/unsloth/llama.cpp WORKDIR /workspace diff --git a/tests/python/test_docker_llama_cuda_backend.py b/tests/python/test_docker_llama_cuda_backend.py new file mode 100644 index 0000000000..34ac3d0fa0 --- /dev/null +++ b/tests/python/test_docker_llama_cuda_backend.py @@ -0,0 +1,89 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-Present the Unsloth team. See /studio/LICENSE.AGPL-3.0 + +"""Regression guard for the llama.cpp CUDA backend inside the Docker image. + +The portable llama.cpp bundle ships libggml-cuda.so and loads it with dlopen +(ggml_backend_dl), but the bundle does NOT carry the CUDA math libraries it +links against, and the CUDA runtime base image only carries libcudart. With no +libcublas on the loader path the backend fails to load SILENTLY: llama.cpp +prints nothing, `--list-devices` comes back empty and every GGUF request runs on +the CPU. Measured on a B200 with gemma-4-E2B UD-Q4_K_XL: 1.6 tok/s instead of +224 tok/s, a 140x regression that no functional test would have caught. + +The Dockerfile therefore has to do two things, and these tests pin both: + * put torch's bundled libcublas on the loader path (ld.so.conf.d, not + LD_LIBRARY_PATH, so llama.cpp's own $ORIGIN libs keep winning); + * fail the build when any non-driver dependency of libggml-cuda.so is still + unresolved, so a CPU-only image can never be published again. + +Static: parses the Dockerfile only. No docker, no GPU, no network. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[2] +DOCKERFILE = REPO_ROOT / "docker" / "Dockerfile" + + +@pytest.fixture(scope="module") +def dockerfile() -> str: + assert DOCKERFILE.is_file(), f"missing {DOCKERFILE}" + return DOCKERFILE.read_text() + + +def test_cublas_dir_is_registered_with_the_loader(dockerfile: str): + conf = re.search( + r"ld\.so\.conf\.d/zz-unsloth-venv\.conf", dockerfile, + ) + assert conf, "the venv loader-config layer disappeared" + block = dockerfile[: conf.end()] + assert "$SP/nvidia/cublas/lib" in block, ( + "libggml-cuda.so links against libcublas, which only exists in the venv's " + "wheel copy; without this entry the CUDA backend fails to dlopen and GGUF " + "silently runs on the CPU" + ) + + +def test_loader_config_is_not_ld_library_path(dockerfile: str): + # LD_LIBRARY_PATH is consulted BEFORE DT_RUNPATH, so it would let the venv's + # copies shadow llama.cpp's own $ORIGIN libs. ld.so.conf.d is consulted after. + assert "ld.so.conf.d/zz-unsloth-venv.conf" in dockerfile + assert not re.search( + r"ENV\s+LD_LIBRARY_PATH=.*site-packages/nvidia", dockerfile, + ), "the venv nvidia libs must not go on LD_LIBRARY_PATH" + + +def test_build_fails_on_an_unresolved_cuda_backend(dockerfile: str): + assert "libggml-cuda.so" in dockerfile, "the CUDA backend guard disappeared" + guard = dockerfile[dockerfile.index("CUDA_SO=") :] + assert "ldd" in guard, "the guard must inspect the backend's dependencies" + assert "not found" in guard + assert "exit 1" in guard, "an unresolved backend must fail the build" + # The driver stub is injected by nvidia-container-toolkit at `docker run + # --gpus`, so it is never resolvable inside the build and must be exempt. + assert re.search(r"grep -v .libcuda\\?\.so\\?\.1", guard), ( + "libcuda.so.1 must be exempt from the guard or every build fails" + ) + + +def test_guard_installs_the_matching_cublas_major(dockerfile: str): + # The amd64 bundle is CUDA 12 and torch already ships libcublas.so.12, but + # the arm64 bundle is CUDA 13. Deriving the major from ldd keeps the two + # legs correct without hardcoding either. + guard = dockerfile[dockerfile.index("CUDA_SO=") :] + assert "nvidia-cublas-cu${major}" in guard, ( + "the guard must install the cublas major the bundle actually asks for" + ) + assert "libcublas" in guard + + +def test_guard_runs_after_the_prebuilt_is_fetched(dockerfile: str): + fetch = dockerfile.index("fetch_llama_prebuilt.py") + guard = dockerfile.index("CUDA_SO=") + assert fetch < guard, "the guard can only inspect a bundle that already exists"