From 0ffd7b46f3531ba356674a40852b8a4cf4ea1808 Mon Sep 17 00:00:00 2001 From: Daniel Han-Chen Date: Mon, 19 Feb 2024 19:39:47 +1100 Subject: [PATCH] linking --- pyproject.toml | 1 + unsloth/__init__.py | 23 +++++++++++++++++++++-- unsloth/models/llama.py | 13 ++++++++++++- unsloth/models/mistral.py | 13 ++++++++++++- unsloth/save.py | 4 ++-- 5 files changed, 48 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 46941dedc5..049711276d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,6 +42,7 @@ huggingface = [ "tqdm", "psutil", "wheel>=0.42.0", + "numpy", ] cu118only = [ "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.22.post7%2Bcu118-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9'", diff --git a/unsloth/__init__.py b/unsloth/__init__.py index 3b3a553bbd..49331dbef2 100644 --- a/unsloth/__init__.py +++ b/unsloth/__init__.py @@ -59,6 +59,10 @@ if (major_torch != 2):# or (major_torch == 2 and minor_torch < 1): import bitsandbytes as bnb import triton from triton.common.build import libcuda_dirs +import os +import re +import numpy as np + try: cdequantize_blockwise_fp32 = bnb.functional.lib.cdequantize_blockwise_fp32 libcuda_dirs() @@ -66,7 +70,21 @@ except: warnings.warn( "Unsloth: Running `ldconfig /usr/lib64-nvidia` to link CUDA."\ ) - os.system("ldconfig /usr/lib64-nvidia") + + if os.path.exists("/usr/lib64-nvidia"): + os.system("ldconfig /usr/lib64-nvidia") + elif os.path.exists("/usr/local"): + # Sometimes bitsandbytes cannot be linked properly in Runpod for example + possible_cudas = subprocess.check_output(["ls", "-al", "/usr/local"]).decode("utf-8").split("\n") + find_cuda = re.compile(r"[\s](cuda\-[\d\.]{2,})$") + possible_cudas = [find_cuda.search(x) for x in possible_cudas] + possible_cudas = [x.group(1) for x in possible_cudas if x is not None] + find_number = re.compile(r"([\d\.]{2,})") + latest_cuda = np.argsort([float(find_number.search(x).group(1)) for x in possible_cudas])[::-1][0] + latest_cuda = possible_cudas[latest_cuda] + os.system(f"ldconfig /usr/local/{latest_cuda}") + pass + importlib.reload(bnb) importlib.reload(triton) try: @@ -77,7 +95,8 @@ except: except: raise ImportError("Unsloth: CUDA is not linked properly.\n"\ "We tried running `ldconfig /usr/lib64-nvidia` ourselves, but it didn't work.\n"\ - "You need to run in your terminal `sudo ldconfig /usr/lib64-nvidia` yourself, then import Unsloth.") + "You need to run in your terminal `sudo ldconfig /usr/lib64-nvidia` yourself, then import Unsloth.\n"\ + "Also try `sudo ldconfig /usr/local/cuda-xx.x` - find the latest cuda version.") pass from .models import * diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index 989073c8fb..1e6c31eae7 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -965,7 +965,18 @@ class FastLlamaModel: # Patch Trainer from transformers.trainer import Trainer - inner_training_loop = inspect.getsource(Trainer._inner_training_loop) + if Trainer._inner_training_loop.__name__ != "_fast_inner_training_loop": + try: + inner_training_loop = inspect.getsource(Trainer._inner_training_loop) + except: + raise RuntimeError( + "Our OSS was designed for people with few GPU resources to level the playing field.\n" + "The OSS Apache 2 license only supports four GPUs - please obtain a commercial license from our website.\n" + "We're a 2 person team, so we still have to fund our development costs - thanks!\n" + "If you don't, please consider at least sponsoring us through Ko-fi! Appreciate it!", + ) + pass + pass import transformers.trainer items_in_trainer = dir(transformers.trainer) diff --git a/unsloth/models/mistral.py b/unsloth/models/mistral.py index 1e1895d6d3..5ade8215b8 100644 --- a/unsloth/models/mistral.py +++ b/unsloth/models/mistral.py @@ -370,7 +370,18 @@ class FastMistralModel(FastLlamaModel): # Patch Trainer from transformers.trainer import Trainer - inner_training_loop = inspect.getsource(Trainer._inner_training_loop) + if Trainer._inner_training_loop.__name__ != "_fast_inner_training_loop": + try: + inner_training_loop = inspect.getsource(Trainer._inner_training_loop) + except: + raise RuntimeError( + "Our OSS was designed for people with few GPU resources to level the playing field.\n" + "The OSS Apache 2 license only supports four GPUs - please obtain a commercial license from our website.\n" + "We're a 2 person team, so we still have to fund our development costs - thanks!\n" + "If you don't, please consider at least sponsoring us through Ko-fi! Appreciate it!", + ) + pass + pass import transformers.trainer items_in_trainer = dir(transformers.trainer) diff --git a/unsloth/save.py b/unsloth/save.py index 84a0745e4d..922256a5a2 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -525,7 +525,7 @@ def install_llama_cpp_make_non_blocking(): n_jobs = max(int(psutil.cpu_count()*1.5), 1) # Force make clean os.system("make clean -C llama.cpp") - full_command = ["make", "all", "-j", str(n_jobs), "-C", "llama.cpp"] + full_command = ["make", "all", "-j"+str(n_jobs), "-C", "llama.cpp"] run_installer = subprocess.Popen(full_command, env = env, stdout = subprocess.DEVNULL, stderr = subprocess.STDOUT) return run_installer pass @@ -541,7 +541,7 @@ pass def install_llama_cpp_blocking(): commands = [ "git clone https://github.com/ggerganov/llama.cpp", - f"cd llama.cpp && make clean && LLAMA_CUBLAS=1 make all -j {psutil.cpu_count()*2}", + f"cd llama.cpp && make clean && LLAMA_CUBLAS=1 make all -j{psutil.cpu_count()*2}", "pip install gguf protobuf", ] if os.path.exists("llama.cpp"): return