From 58c640dfe54f115fe169e13665e35b153a122b55 Mon Sep 17 00:00:00 2001 From: Daniel Han <23090290+danielhanchen@users.noreply.github.com> Date: Wed, 18 Mar 2026 04:58:10 +0000 Subject: [PATCH 1/4] Fix torch CUDA symbol errors caused by user LD_LIBRARY_PATH When a user has LD_LIBRARY_PATH pointing at system CUDA libs (e.g. /usr/local/cuda-13/lib64), the dynamic linker loads those instead of the CUDA libs bundled with the torch wheel (in nvidia/*/lib/). This causes symbol version mismatches and crashes on import. Fix: before importing anything, detect torch's bundled CUDA lib paths (without importing torch itself) and prepend them to LD_LIBRARY_PATH so they take priority. Then re-exec so the dynamic linker picks up the corrected path. A sentinel env var prevents infinite re-exec. The user's original paths are preserved (just deprioritized), so non-torch tools that need system CUDA still work. --- studio/backend/run.py | 61 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/studio/backend/run.py b/studio/backend/run.py index 2f064ab92e..d2e9677bab 100644 --- a/studio/backend/run.py +++ b/studio/backend/run.py @@ -12,6 +12,67 @@ import sys # Suppress annoying C-level dependency warnings globally (e.g. SwigPyPacked) os.environ["PYTHONWARNINGS"] = "ignore" + +def _fix_torch_cuda_ld_path(): + """Prepend torch's bundled CUDA libs to LD_LIBRARY_PATH. + + PyTorch wheels ship their own CUDA runtime (libcudart, libcublas, etc.) + inside ``site-packages/nvidia/*/lib/``. On Linux the dynamic linker + checks LD_LIBRARY_PATH **before** the RUNPATH baked into the .so files, + so a user's pre-existing LD_LIBRARY_PATH pointing at a different system + CUDA (e.g. /usr/local/cuda-13/lib64) will shadow torch's libs and cause + symbol-version errors at import time. + + Fix: detect torch's lib dirs (without importing torch) and prepend them + so they take priority. Returns True if LD_LIBRARY_PATH was changed. + """ + if sys.platform == "win32": + return False # Windows uses PATH, not LD_LIBRARY_PATH + ld_path = os.environ.get("LD_LIBRARY_PATH", "") + if not ld_path: + return False + try: + import importlib.util + spec = importlib.util.find_spec("torch") + if not spec or not spec.origin: + return False + torch_dir = os.path.dirname(spec.origin) + site_pkgs = os.path.dirname(torch_dir) + nvidia_dir = os.path.join(site_pkgs, "nvidia") + + lib_dirs = [] + torch_lib = os.path.join(torch_dir, "lib") + if os.path.isdir(torch_lib): + lib_dirs.append(torch_lib) + if os.path.isdir(nvidia_dir): + for sub in sorted(os.listdir(nvidia_dir)): + lib = os.path.join(nvidia_dir, sub, "lib") + if os.path.isdir(lib): + lib_dirs.append(lib) + if not lib_dirs: + return False + + # Already at the front -- nothing to do + existing = ld_path.split(":") + if existing[:len(lib_dirs)] == lib_dirs: + return False + + # Prepend torch dirs, deduplicate + torch_set = set(lib_dirs) + cleaned = [p for p in existing if p not in torch_set] + os.environ["LD_LIBRARY_PATH"] = ":".join(lib_dirs + cleaned) + return True + except Exception: + return False + + +# Fix LD_LIBRARY_PATH before any torch/CUDA libs get loaded. +# If we changed it, re-exec so the dynamic linker sees the new value. +_LD_FIXED_SENTINEL = "_UNSLOTH_STUDIO_LD_FIXED" +if _LD_FIXED_SENTINEL not in os.environ and _fix_torch_cuda_ld_path(): + os.environ[_LD_FIXED_SENTINEL] = "1" + os.execv(sys.executable, [sys.executable] + sys.argv) + from pathlib import Path # Add the backend directory to Python path From 35f3465d869b510e9e0d4bd01aadaabe08e98634 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 18 Mar 2026 05:00:40 +0000 Subject: [PATCH 2/4] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- studio/backend/run.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/studio/backend/run.py b/studio/backend/run.py index d2e9677bab..38d7196475 100644 --- a/studio/backend/run.py +++ b/studio/backend/run.py @@ -33,6 +33,7 @@ def _fix_torch_cuda_ld_path(): return False try: import importlib.util + spec = importlib.util.find_spec("torch") if not spec or not spec.origin: return False @@ -54,7 +55,7 @@ def _fix_torch_cuda_ld_path(): # Already at the front -- nothing to do existing = ld_path.split(":") - if existing[:len(lib_dirs)] == lib_dirs: + if existing[: len(lib_dirs)] == lib_dirs: return False # Prepend torch dirs, deduplicate From e235df6d2a850eccb164b8a50b7d1f0759ea54b7 Mon Sep 17 00:00:00 2001 From: Daniel Han <23090290+danielhanchen@users.noreply.github.com> Date: Wed, 18 Mar 2026 05:13:30 +0000 Subject: [PATCH 3/4] Fix review issues: no module-level execv, Linux-only, use sys.orig_argv Addresses all reviewer feedback: 1. Moved os.execv out of module-level code into a helper function _maybe_reexec_for_cuda_ld_path() that is called explicitly from run_server() and __main__. Importing run.py no longer replaces the host process -- safe for CLI, notebooks, tests, and embedders. 2. Changed platform guard from "not win32" to "linux only", since LD_LIBRARY_PATH is a Linux-specific linker mechanism. 3. Use sys.orig_argv (Python 3.10+) when available to preserve the original interpreter invocation (python -m, -c, -X flags, etc.). Falls back to [sys.executable] + sys.argv for older Python. --- studio/backend/run.py | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/studio/backend/run.py b/studio/backend/run.py index 38d7196475..4875b0a84e 100644 --- a/studio/backend/run.py +++ b/studio/backend/run.py @@ -26,8 +26,8 @@ def _fix_torch_cuda_ld_path(): Fix: detect torch's lib dirs (without importing torch) and prepend them so they take priority. Returns True if LD_LIBRARY_PATH was changed. """ - if sys.platform == "win32": - return False # Windows uses PATH, not LD_LIBRARY_PATH + if sys.platform != "linux": + return False ld_path = os.environ.get("LD_LIBRARY_PATH", "") if not ld_path: return False @@ -67,12 +67,24 @@ def _fix_torch_cuda_ld_path(): return False -# Fix LD_LIBRARY_PATH before any torch/CUDA libs get loaded. -# If we changed it, re-exec so the dynamic linker sees the new value. _LD_FIXED_SENTINEL = "_UNSLOTH_STUDIO_LD_FIXED" -if _LD_FIXED_SENTINEL not in os.environ and _fix_torch_cuda_ld_path(): + + +def _maybe_reexec_for_cuda_ld_path(): + """Re-exec once so the dynamic linker sees corrected LD_LIBRARY_PATH. + + Must only be called from a true entry point (``if __name__ == "__main__"`` + or an explicit startup function), never at module import time, because + os.execv replaces the entire process. + """ + if _LD_FIXED_SENTINEL in os.environ: + return + if not _fix_torch_cuda_ld_path(): + return os.environ[_LD_FIXED_SENTINEL] = "1" - os.execv(sys.executable, [sys.executable] + sys.argv) + argv = getattr(sys, "orig_argv", None) or [sys.executable] + sys.argv + os.execv(sys.executable, argv) + from pathlib import Path @@ -237,6 +249,8 @@ def run_server( """ global _server, _shutdown_event + _maybe_reexec_for_cuda_ld_path() + import nest_asyncio nest_asyncio.apply() @@ -300,6 +314,8 @@ def run_server( # For direct execution (also invoked by CLI via os.execvp / subprocess) if __name__ == "__main__": + _maybe_reexec_for_cuda_ld_path() + import argparse import signal From 2773f98357cd93124322623c1b1e15fa75a02689 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Mon, 6 Jul 2026 10:32:31 +0000 Subject: [PATCH 4/4] Gate run_server re-exec behind allow_reexec so embedders are not restarted run_server is a library/embed entry point (colab.start calls it directly), so the unconditional os.execv for the torch CUDA LD_LIBRARY_PATH fix would replace the live Colab/Jupyter kernel and drop in-memory state. Gate the re-exec behind a new allow_reexec flag defaulting to False; the run.py __main__ path already re-execs before calling run_server, and the unsloth_cli studio/ui entrypoints opt in with allow_reexec=True so the CLI keeps the CUDA LD fix. --- studio/backend/run.py | 14 +++++++++++++- unsloth_cli/commands/studio.py | 2 ++ unsloth_cli/commands/ui.py | 2 ++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/studio/backend/run.py b/studio/backend/run.py index 4875b0a84e..0ae1b4c697 100644 --- a/studio/backend/run.py +++ b/studio/backend/run.py @@ -232,6 +232,7 @@ def run_server( port: int = 8000, frontend_path: Path = Path(__file__).resolve().parent.parent / "frontend" / "dist", silent: bool = False, + allow_reexec: bool = False, ): """ Start the FastAPI server. @@ -241,6 +242,11 @@ def run_server( port: Port to bind to (auto-increments if in use) frontend_path: Path to frontend build directory (optional) silent: Suppress startup messages + allow_reexec: Re-exec the process (os.execv) to apply the torch CUDA + LD_LIBRARY_PATH fix. Enable ONLY from true CLI/process entrypoints. + Embedders (e.g. Colab via colab.start) must leave this False, + otherwise the re-exec replaces the notebook kernel and drops + in-memory state. Note: Signal handlers are NOT registered here so that embedders @@ -249,7 +255,13 @@ def run_server( """ global _server, _shutdown_event - _maybe_reexec_for_cuda_ld_path() + # Only re-exec when invoked as a real CLI/process entrypoint. Embedders + # (e.g. Colab via colab.start -> run_server) keep allow_reexec=False, + # because os.execv would replace the live notebook kernel and drop + # in-memory state. CLI entrypoints (run.py __main__, unsloth_cli in-venv + # fallback) opt in explicitly. + if allow_reexec: + _maybe_reexec_for_cuda_ld_path() import nest_asyncio diff --git a/unsloth_cli/commands/studio.py b/unsloth_cli/commands/studio.py index 829f086b7d..11d8a1946b 100644 --- a/unsloth_cli/commands/studio.py +++ b/unsloth_cli/commands/studio.py @@ -138,6 +138,8 @@ def studio_default( port = port, frontend_path = frontend, silent = silent, + # CLI entrypoint: apply the torch CUDA LD_LIBRARY_PATH fix via re-exec. + allow_reexec = True, ) from studio.backend.run import _shutdown_event diff --git a/unsloth_cli/commands/ui.py b/unsloth_cli/commands/ui.py index eb6a8a69e8..a35c0d3a68 100644 --- a/unsloth_cli/commands/ui.py +++ b/unsloth_cli/commands/ui.py @@ -85,6 +85,8 @@ def ui( port = port, frontend_path = frontend, silent = silent, + # CLI entrypoint: apply the torch CUDA LD_LIBRARY_PATH fix via re-exec. + allow_reexec = True, ) from studio.backend.run import _shutdown_event