diff --git a/unsloth/__init__.py b/unsloth/__init__.py index 19b433a680..19639a11c7 100644 --- a/unsloth/__init__.py +++ b/unsloth/__init__.py @@ -208,9 +208,11 @@ elif DEVICE_TYPE == "xpu": # Backwards compatibility: some notebooks import `unsloth.is_bf16_supported`. # Ensure it exists on all backends (HIP / XPU) and has a stable signature. if "is_bf16_supported" not in globals(): + def is_bf16_supported(including_emulation = False): return SUPPORTS_BFLOAT16 + # For Gradio HF Spaces? # if "SPACE_AUTHOR_NAME" not in os.environ and "SPACE_REPO_NAME" not in os.environ: import triton diff --git a/unsloth/kernels/utils.py b/unsloth/kernels/utils.py index 63b1966b32..29d47c35e3 100644 --- a/unsloth/kernels/utils.py +++ b/unsloth/kernels/utils.py @@ -110,6 +110,7 @@ def calculate_settings( HAS_CUDA_STREAM = False try: import bitsandbytes as bnb + # https://github.com/bitsandbytes-foundation/bitsandbytes/pull/1330/files HAS_CUDA_STREAM = Version(bnb.__version__) > Version("0.43.3") get_ptr = bnb.functional.get_ptr @@ -197,8 +198,12 @@ if bnb is not None: cgemm_4bit_inference_naive_fp16 = bnb.functional.lib.cgemv_4bit_inference_fp16 cgemm_4bit_inference_naive_bf16 = bnb.functional.lib.cgemv_4bit_inference_bf16 else: - cgemm_4bit_inference_naive_fp16 = bnb.functional.lib.cgemm_4bit_inference_naive_fp16 - cgemm_4bit_inference_naive_bf16 = bnb.functional.lib.cgemm_4bit_inference_naive_bf16 + cgemm_4bit_inference_naive_fp16 = ( + bnb.functional.lib.cgemm_4bit_inference_naive_fp16 + ) + cgemm_4bit_inference_naive_bf16 = ( + bnb.functional.lib.cgemm_4bit_inference_naive_bf16 + ) else: cdequantize_blockwise_fp32 = None cdequantize_blockwise_fp16_nf4 = None @@ -207,7 +212,6 @@ else: cgemm_4bit_inference_naive_bf16 = None - torch_device_stream = ( torch.xpu.current_stream if DEVICE_TYPE == "xpu" else torch.cuda.current_stream ) diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 3d48438800..4c75b9f64f 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -1914,7 +1914,10 @@ def patch_fast_lora(): try: import peft.tuners.lora.bnb except Exception as e: - print("Unsloth: bitsandbytes/peft bnb not available - skipping 4bit LoRA patch.", repr(e)) + print( + "Unsloth: bitsandbytes/peft bnb not available - skipping 4bit LoRA patch.", + repr(e), + ) return peft.tuners.lora.bnb.Linear4bit.forward = fast_lora_forward diff --git a/unsloth/models/granite.py b/unsloth/models/granite.py index e28c21233d..c40be8dfdb 100644 --- a/unsloth/models/granite.py +++ b/unsloth/models/granite.py @@ -30,6 +30,7 @@ from .llama import ( LlamaLinearScalingRotaryEmbedding, ) from .mistral import * + try: from bitsandbytes.nn import Linear4bit as Bnb_Linear4bit except Exception: @@ -40,9 +41,7 @@ try: except Exception: Peft_Linear4bit = None -_BNB_LINEAR_TYPES = tuple( - t for t in (Bnb_Linear4bit, Peft_Linear4bit) if t is not None -) +_BNB_LINEAR_TYPES = tuple(t for t in (Bnb_Linear4bit, Peft_Linear4bit) if t is not None) try: from transformers.models.granite.modeling_granite import ( diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index 63390a4927..9d3ab0ecaf 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -3004,7 +3004,11 @@ class FastLlamaModel: # PEFT API compatibility: only pass kwargs supported by the installed peft version. try: import inspect as _inspect - if "ensure_weight_tying" not in _inspect.signature(LoraConfig.__init__).parameters: + + if ( + "ensure_weight_tying" + not in _inspect.signature(LoraConfig.__init__).parameters + ): arguments.pop("ensure_weight_tying", None) except Exception: arguments.pop("ensure_weight_tying", None) diff --git a/unsloth/save.py b/unsloth/save.py index 0187ded794..5f26e0a210 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -23,6 +23,7 @@ from unsloth_zoo.llama_cpp import ( check_llama_cpp, _download_convert_hf_to_gguf, ) + try: from bitsandbytes.nn import Linear4bit as Bnb_Linear4bit except Exception: @@ -65,7 +66,9 @@ except: from pathlib import Path from peft import PeftModelForCausalLM, PeftModel -_MERGE_LORA_LINEAR_TYPES = tuple(t for t in (Bnb_Linear4bit, Peft_Linear4bit, Peft_Linear) if t is not None) +_MERGE_LORA_LINEAR_TYPES = tuple( + t for t in (Bnb_Linear4bit, Peft_Linear4bit, Peft_Linear) if t is not None +) __all__ = [ "print_quantization_methods",