linking

pyproject.toml

@@ -42,6 +42,7 @@ huggingface = [
     "tqdm",
     "psutil",
     "wheel>=0.42.0",
+    "numpy",
 ]
 cu118only = [
     "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.22.post7%2Bcu118-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9'",

unsloth/__init__.py

@@ -59,6 +59,10 @@ if (major_torch != 2):# or (major_torch == 2 and minor_torch < 1):
 import bitsandbytes as bnb
 import triton
 from triton.common.build import libcuda_dirs
+import os
+import re
+import numpy as np
+
 try:
     cdequantize_blockwise_fp32 = bnb.functional.lib.cdequantize_blockwise_fp32
     libcuda_dirs()
@@ -66,7 +70,21 @@ except:
     warnings.warn(
         "Unsloth: Running `ldconfig /usr/lib64-nvidia` to link CUDA."\
     )
-    os.system("ldconfig /usr/lib64-nvidia")
+
+    if os.path.exists("/usr/lib64-nvidia"):
+        os.system("ldconfig /usr/lib64-nvidia")
+    elif os.path.exists("/usr/local"):
+        # Sometimes bitsandbytes cannot be linked properly in Runpod for example
+        possible_cudas = subprocess.check_output(["ls", "-al", "/usr/local"]).decode("utf-8").split("\n")
+        find_cuda = re.compile(r"[\s](cuda\-[\d\.]{2,})$")
+        possible_cudas = [find_cuda.search(x) for x in possible_cudas]
+        possible_cudas = [x.group(1) for x in possible_cudas if x is not None]
+        find_number = re.compile(r"([\d\.]{2,})")
+        latest_cuda = np.argsort([float(find_number.search(x).group(1)) for x in possible_cudas])[::-1][0]
+        latest_cuda = possible_cudas[latest_cuda]
+        os.system(f"ldconfig /usr/local/{latest_cuda}")
+    pass
+
     importlib.reload(bnb)
     importlib.reload(triton)
     try:
@@ -77,7 +95,8 @@ except:
     except:
         raise ImportError("Unsloth: CUDA is not linked properly.\n"\
                           "We tried running `ldconfig /usr/lib64-nvidia` ourselves, but it didn't work.\n"\
-                          "You need to run in your terminal `sudo ldconfig /usr/lib64-nvidia` yourself, then import Unsloth.")
+                          "You need to run in your terminal `sudo ldconfig /usr/lib64-nvidia` yourself, then import Unsloth.\n"\
+                          "Also try `sudo ldconfig /usr/local/cuda-xx.x` - find the latest cuda version.")
 pass
 
 from .models import *

unsloth/models/llama.py

@@ -965,7 +965,18 @@ class FastLlamaModel:
 
         # Patch Trainer
         from transformers.trainer import Trainer
-        inner_training_loop = inspect.getsource(Trainer._inner_training_loop)
+        if Trainer._inner_training_loop.__name__ != "_fast_inner_training_loop":
+            try:
+                inner_training_loop = inspect.getsource(Trainer._inner_training_loop)
+            except:
+                raise RuntimeError(
+                    "Our OSS was designed for people with few GPU resources to level the playing field.\n"
+                    "The OSS Apache 2 license only supports four GPUs - please obtain a commercial license from our website.\n"
+                    "We're a 2 person team, so we still have to fund our development costs - thanks!\n"
+                    "If you don't, please consider at least sponsoring us through Ko-fi! Appreciate it!",
+                )
+            pass
+        pass
 
         import transformers.trainer
         items_in_trainer = dir(transformers.trainer)

unsloth/models/mistral.py

@@ -370,7 +370,18 @@ class FastMistralModel(FastLlamaModel):
 
         # Patch Trainer
         from transformers.trainer import Trainer
-        inner_training_loop = inspect.getsource(Trainer._inner_training_loop)
+        if Trainer._inner_training_loop.__name__ != "_fast_inner_training_loop":
+            try:
+                inner_training_loop = inspect.getsource(Trainer._inner_training_loop)
+            except:
+                raise RuntimeError(
+                    "Our OSS was designed for people with few GPU resources to level the playing field.\n"
+                    "The OSS Apache 2 license only supports four GPUs - please obtain a commercial license from our website.\n"
+                    "We're a 2 person team, so we still have to fund our development costs - thanks!\n"
+                    "If you don't, please consider at least sponsoring us through Ko-fi! Appreciate it!",
+                )
+            pass
+        pass
 
         import transformers.trainer
         items_in_trainer = dir(transformers.trainer)

unsloth/save.py

@@ -525,7 +525,7 @@ def install_llama_cpp_make_non_blocking():
     n_jobs = max(int(psutil.cpu_count()*1.5), 1)
     # Force make clean
     os.system("make clean -C llama.cpp")
-    full_command = ["make", "all", "-j", str(n_jobs), "-C", "llama.cpp"]
+    full_command = ["make", "all", "-j"+str(n_jobs), "-C", "llama.cpp"]
     run_installer = subprocess.Popen(full_command, env = env, stdout = subprocess.DEVNULL, stderr = subprocess.STDOUT)
     return run_installer
 pass
@@ -541,7 +541,7 @@ pass
 def install_llama_cpp_blocking():
     commands = [
         "git clone https://github.com/ggerganov/llama.cpp",
-        f"cd llama.cpp && make clean && LLAMA_CUBLAS=1 make all -j {psutil.cpu_count()*2}",
+        f"cd llama.cpp && make clean && LLAMA_CUBLAS=1 make all -j{psutil.cpu_count()*2}",
         "pip install gguf protobuf",
     ]
     if os.path.exists("llama.cpp"): return