diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 22ce2fd9e5..f2d2e9ee70 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -905,6 +905,26 @@ if Version(peft_version) < Version("0.12.0"): pass # ============================================= +import importlib +global USE_MODELSCOPE +USE_MODELSCOPE = os.environ.get("UNSLOTH_USE_MODELSCOPE", "0") == "1" +if USE_MODELSCOPE: + if importlib.util.find_spec("modelscope") is None: + raise ImportError(f'You are using the modelscope hub, please install modelscope by `pip install modelscope -U`') + pass +pass + +import socket +@functools.lru_cache(1) +def has_internet(host = "8.8.8.8", port = 53, timeout = 3): + if os.environ.get("TRANSFORMERS_OFFLINE", "0") == "1": return False + try: + socket.setdefaulttimeout(timeout) + socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect((host, port)) + return True + except socket.error as ex: + return False +pass import psutil def _get_statistics(statistics = None, force_download = True): @@ -912,61 +932,81 @@ def _get_statistics(statistics = None, force_download = True): # We simply download a README.md file from HF - all data is made public. # This is simply so we can check if some envs are broken or not. # You can disable this by commenting the below out - try: - n_cpus = psutil.cpu_count(logical = False) - keynames = "\n" + "\n".join(os.environ.keys()) - if statistics is not None: pass - elif "\nCOLAB_" in keynames and n_cpus == 1: statistics = "colab" - elif "\nCOLAB_" in keynames: statistics = "colabpro" - elif "\nKAGGLE_" in keynames: statistics = "kaggle" - elif "\nRUNPOD_" in keynames: statistics = "runpod" - elif "\nAWS_" in keynames: statistics = "aws" - elif "\nAZURE_" in keynames: statistics = "azure" - # elif "\nK_" in keynames or "\nFUNCTION_" in keynames: statistics = "gcp" - elif "\nINVOCATION_ID" in keynames: statistics = "lambda" - # else: statistics = "other" - else: - def try_vllm_check(): - vendor_files = ( - "/sys/class/dmi/id/product_version", - "/sys/class/dmi/id/bios_vendor", - "/sys/class/dmi/id/product_name", - "/sys/class/dmi/id/chassis_asset_tag", - "/sys/class/dmi/id/sys_vendor", - ) - from pathlib import Path - for vendor_file in vendor_files: - path = Path(vendor_file) - if path.is_file(): - file_content = path.read_text().lower() - if "amazon" in file_content: return "aws" - elif "microsoft corporation" in file_content: return "azure" - elif "google" in file_content: return "gcp" - return "other" - pass - try: statistics = try_vllm_check() - except: statistics = "other" - pass - if statistics is not None: - from transformers import AutoModelForCausalLM - stats_model = AutoModelForCausalLM.from_pretrained( - f"unslothai/{statistics}", - force_download = force_download, + n_cpus = psutil.cpu_count(logical = False) + keynames = "\n" + "\n".join(os.environ.keys()) + # Check modelscope for down detection + global USE_MODELSCOPE + USE_MODELSCOPE = os.environ.get("UNSLOTH_USE_MODELSCOPE", "0") == "1" + + if statistics is not None: pass + elif "\nCOLAB_" in keynames and n_cpus == 1: statistics = "colab" + elif "\nCOLAB_" in keynames: statistics = "colabpro" + elif "\nKAGGLE_" in keynames: statistics = "kaggle" + elif "\nRUNPOD_" in keynames: statistics = "runpod" + elif "\nAWS_" in keynames: statistics = "aws" + elif "\nAZURE_" in keynames: statistics = "azure" + # elif "\nK_" in keynames or "\nFUNCTION_" in keynames: statistics = "gcp" + elif "\nINVOCATION_ID" in keynames: statistics = "lambda" + # else: statistics = "other" + else: + def try_vllm_check(): + vendor_files = ( + "/sys/class/dmi/id/product_version", + "/sys/class/dmi/id/bios_vendor", + "/sys/class/dmi/id/product_name", + "/sys/class/dmi/id/chassis_asset_tag", + "/sys/class/dmi/id/sys_vendor", ) - del stats_model + from pathlib import Path + for vendor_file in vendor_files: + path = Path(vendor_file) + if path.is_file(): + file_content = path.read_text().lower() + if "amazon" in file_content: return "aws" + elif "microsoft corporation" in file_content: return "azure" + elif "google" in file_content: return "gcp" + return "other" pass - except: + try: statistics = try_vllm_check() + except: statistics = "other" + pass + if statistics is not None: + import tempfile + from huggingface_hub import snapshot_download + from unsloth_zoo.rl_environments import execute_with_time_limit + if has_internet(): + @execute_with_time_limit(120) + def stats_check(): + with tempfile.TemporaryDirectory(ignore_cleanup_errors = True) as f: + snapshot_download(f"unslothai/{statistics}", force_download = True, cache_dir = f, local_dir = f) + try: + stats_check() + except TimeoutError: + raise TimeoutError( + "Unsloth: HuggingFace seems to be down after trying for 120 seconds :(\n"\ + "Check https://status.huggingface.co/ for more details.\n"\ + "As a temporary measure, use modelscope with the same model name ie:\n"\ + "```\n"\ + "pip install modelscope\n"\ + "import os; os.environ['UNSLOTH_USE_MODELSCOPE'] = '1'\n"\ + "from unsloth import FastLanguageModel\n"\ + "model = FastLanguageModel.from_pretrained('unsloth/gpt-oss-20b')\n"\ + "```" + ) pass + pass pass -def get_statistics(): +def get_statistics(local_files_only = False): # We log some basic stats about which environment is being used. + # This is also to check if HuggingFace is down or not! # We simply download a README.md file from HF - all data is made public. # This is simply so we can check if some envs are broken or not. # You can disable this by setting UNSLOTH_DISABLE_STATISTICS import os if "UNSLOTH_DISABLE_STATISTICS" in os.environ: return + if local_files_only: return from huggingface_hub.utils import disable_progress_bars, enable_progress_bars, are_progress_bars_disabled disabled = False if not are_progress_bars_disabled(): @@ -975,24 +1015,17 @@ def get_statistics(): pass _get_statistics(None) _get_statistics("repeat", force_download = False) - try: - vram = torch.cuda.get_device_properties(0).total_memory / 1024 / 1024 / 1024 - if vram <= 8 : vram = 8 - elif vram <= 16: vram = 16 - elif vram <= 20: vram = 20 - elif vram <= 24: vram = 24 - elif vram <= 40: vram = 40 - elif vram <= 48: vram = 48 - elif vram <= 80: vram = 80 - else: vram = 96 - _get_statistics(f"vram-{vram}") - except: - pass - pass - try: - _get_statistics(f"{DEVICE_COUNT if DEVICE_COUNT <= 8 else 9}") - except: - pass + vram = torch.cuda.get_device_properties(0).total_memory / 1024 / 1024 / 1024 + if vram <= 8 : vram = 8 + elif vram <= 16: vram = 16 + elif vram <= 20: vram = 20 + elif vram <= 24: vram = 24 + elif vram <= 40: vram = 40 + elif vram <= 48: vram = 48 + elif vram <= 80: vram = 80 + else: vram = 96 + _get_statistics(f"vram-{vram}") + _get_statistics(f"{DEVICE_COUNT if DEVICE_COUNT <= 8 else 9}") if disabled: enable_progress_bars() pass @@ -1592,14 +1625,6 @@ for j, function in enumerate(functions): except: continue pass -import importlib -USE_MODELSCOPE = os.environ.get("UNSLOTH_USE_MODELSCOPE", "0") == "1" -if USE_MODELSCOPE: - if importlib.util.find_spec("modelscope") is None: - raise ImportError(f'You are using the modelscope hub, please install modelscope by `pip install modelscope -U`') - pass -pass - def validate_loftq_config(loftq_config, lora_dropout, bias, init_lora_weights, model): from peft import LoraConfig diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index 535537a3a1..d154dfe20a 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -1205,7 +1205,7 @@ def CausalLM_fast_forward(fast_forward_inference): # < 1024 Normal Unsloth uses less VRAM! if DEVICE_TYPE == "hip": # [TODO] AMD GPUs fail on chunked_cross_entropy loss! - # RuntimeError: Triton Error [HIP]: Code: 1, Messsage: invalid argument + # RuntimeError: Triton Error [HIP]: Code: 1, Messsage: invalid argument RETURN_LOGITS = False elif bsz*q_len <= 1024: RETURN_LOGITS = True @@ -1217,6 +1217,8 @@ def CausalLM_fast_forward(fast_forward_inference): if self.config.model_type == "falcon_h1": hidden_states = hidden_states * self.config.lm_head_multiplier + ### DISABLED since T4 breaks + # OutOfResources: out of resource: shared memory, Required: 98304, Hardware limit: 65536. Reducing block sizes or `num_stages` may help. # loss = fused_linear_cross_entropy( # hidden_states = hidden_states, # lm_weight = lm_head, @@ -1242,11 +1244,11 @@ def CausalLM_fast_forward(fast_forward_inference): return (loss,) + output if loss is not None else output output = CausalLMOutputWithPast( - loss=loss, - logits=EMPTY_LOGITS, - past_key_values=outputs.past_key_values, - hidden_states=outputs.hidden_states, - attentions=outputs.attentions, + loss = loss, + logits = EMPTY_LOGITS, + past_key_values= outputs.past_key_values, + hidden_states = outputs.hidden_states, + attentions = outputs.attentions, ) return output pass @@ -1922,7 +1924,8 @@ class FastLlamaModel: if old_hf_transfer != "0": os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1" model_patcher.pre_patch() - get_statistics() # For debugging - we use a download counter to see if environments are not breaking + # For debugging - we use a download counter to see if environments are not breaking or if HF is down + get_statistics(kwargs.get("local_files_only", False)) if dtype is None: dtype = torch.float16 if not SUPPORTS_BFLOAT16 else torch.bfloat16 diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index 4c6c91f6fc..e17fb58a46 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -210,10 +210,14 @@ class FastLanguageModel(FastLlamaModel): model_name = get_model_name(model_name, load_in_4bit) # Check if pre-quantized models are allowed # For eg AMD GPUs need blocksize = 128, but our pre-quants are blocksize = 64 - if not ALLOW_PREQUANTIZED_MODELS and model_name.endswith(("-unsloth-bnb-4bit", "-bnb-4bit")): - model_name = model_name.removesuffix("-unsloth-bnb-4bit") - model_name = model_name.removesuffix("-bnb-4bit") - pass + if not ALLOW_PREQUANTIZED_MODELS and model_name.lower().endswith(("-unsloth-bnb-4bit", "-bnb-4bit")): + model_name = model_name.lower().removesuffix("-unsloth-bnb-4bit") + model_name = model_name.lower().removesuffix("-bnb-4bit") + # Change -BF16 to all False for 4bit, 8bit etc + if model_name.lower().endswith("-bf16"): + load_in_4bit = False + load_in_8bit = False + load_in_16bit = True if USE_MODELSCOPE and not os.path.exists(model_name): from modelscope import snapshot_download @@ -327,10 +331,15 @@ class FastLanguageModel(FastLlamaModel): model_name = get_model_name(model_name, load_in_4bit) # Check if pre-quantized models are allowed # For eg AMD GPUs need blocksize = 128, but our pre-quants are blocksize = 64 - if not ALLOW_PREQUANTIZED_MODELS and model_name.endswith(("-unsloth-bnb-4bit", "-bnb-4bit")): - model_name = model_name.removesuffix("-unsloth-bnb-4bit") - model_name = model_name.removesuffix("-bnb-4bit") - pass + if not ALLOW_PREQUANTIZED_MODELS and model_name.lower().endswith(("-unsloth-bnb-4bit", "-bnb-4bit")): + model_name = model_name.lower().removesuffix("-unsloth-bnb-4bit") + model_name = model_name.lower().removesuffix("-bnb-4bit") + # Change -BF16 to all False for 4bit, 8bit etc + if model_name.lower().endswith("-bf16"): + load_in_4bit = False + load_in_8bit = False + load_in_16bit = True + model_config = AutoConfig.from_pretrained( model_name, token = token, @@ -649,10 +658,14 @@ class FastModel(FastBaseModel): model_name = get_model_name(model_name, load_in_4bit) # Check if pre-quantized models are allowed # For eg AMD GPUs need blocksize = 128, but our pre-quants are blocksize = 64 - if not ALLOW_PREQUANTIZED_MODELS and model_name.endswith(("-unsloth-bnb-4bit", "-bnb-4bit")): - model_name = model_name.removesuffix("-unsloth-bnb-4bit") - model_name = model_name.removesuffix("-bnb-4bit") - pass + if not ALLOW_PREQUANTIZED_MODELS and model_name.lower().endswith(("-unsloth-bnb-4bit", "-bnb-4bit")): + model_name = model_name.lower().removesuffix("-unsloth-bnb-4bit") + model_name = model_name.lower().removesuffix("-bnb-4bit") + # Change -BF16 to all False for 4bit, 8bit etc + if model_name.lower().endswith("-bf16"): + load_in_4bit = False + load_in_8bit = False + load_in_16bit = True # Check modelscope if USE_MODELSCOPE and not os.path.exists(model_name): @@ -870,10 +883,15 @@ class FastModel(FastBaseModel): model_name = get_model_name(model_name, load_in_4bit) # Check if pre-quantized models are allowed # For eg AMD GPUs need blocksize = 128, but our pre-quants are blocksize = 64 - if not ALLOW_PREQUANTIZED_MODELS and model_name.endswith(("-unsloth-bnb-4bit", "-bnb-4bit")): - model_name = model_name.removesuffix("-unsloth-bnb-4bit") - model_name = model_name.removesuffix("-bnb-4bit") - pass + if not ALLOW_PREQUANTIZED_MODELS and model_name.lower().endswith(("-unsloth-bnb-4bit", "-bnb-4bit")): + model_name = model_name.lower().removesuffix("-unsloth-bnb-4bit") + model_name = model_name.lower().removesuffix("-bnb-4bit") + # Change -BF16 to all False for 4bit, 8bit etc + if model_name.lower().endswith("-bf16"): + load_in_4bit = False + load_in_8bit = False + load_in_16bit = True + model_config = AutoConfig.from_pretrained( model_name, token = token, diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py index f2bd7c306b..d6322d77f2 100644 --- a/unsloth/models/vision.py +++ b/unsloth/models/vision.py @@ -416,7 +416,8 @@ class FastBaseModel: pass if old_hf_transfer != "0": os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1" - get_statistics() # For debugging - we use a download counter to see if environments are not breaking + # For debugging - we use a download counter to see if environments are not breaking or if HF is down + get_statistics(kwargs.get("local_files_only", False)) if dtype is None: dtype = torch.float16 if not SUPPORTS_BFLOAT16 else torch.bfloat16 diff --git a/unsloth/save.py b/unsloth/save.py index 506c8a68f1..a62d63fc86 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -2565,10 +2565,10 @@ def unsloth_save_pretrained_torchao( """ # first merge the lora weights arguments = dict(locals()) - arguments["model"] = self - arguments["tokenizer"] = tokenizer - arguments["push_to_hub"] = False # We save ourselves - arguments["save_method"] = "merged_16bit" # Must be 16bit + arguments["model"] = self + arguments["tokenizer"] = tokenizer + arguments["push_to_hub"] = False # We save ourselves + arguments["save_method"] = "merged_16bit" # Must be 16bit del arguments["self"] del arguments["torchao_config"] @@ -2722,7 +2722,7 @@ def patch_saving_functions(model, vision = False): model.save_pretrained_merged = types.MethodType(unsloth_generic_save_pretrained_merged, model) model.push_to_hub_gguf = types.MethodType(unsloth_push_to_hub_gguf, model) model.save_pretrained_gguf = types.MethodType(unsloth_save_pretrained_gguf, model) - model.save_pretrained_torchao = types.MethodType(unsloth_save_pretrained_torchao, model) + model.save_pretrained_torchao = types.MethodType(unsloth_save_pretrained_torchao, model) model.push_to_hub_ggml = types.MethodType(unsloth_convert_lora_to_ggml_and_push_to_hub, model) model.save_pretrained_ggml = types.MethodType(unsloth_convert_lora_to_ggml_and_save_locally, model) pass @@ -2732,7 +2732,7 @@ def patch_saving_functions(model, vision = False): model.save_pretrained_merged = types.MethodType(unsloth_generic_save_pretrained_merged, model) model.push_to_hub_gguf = types.MethodType(unsloth_push_to_hub_gguf, model) model.save_pretrained_gguf = types.MethodType(unsloth_save_pretrained_gguf, model) - model.save_pretrained_torchao = types.MethodType(unsloth_save_pretrained_torchao, model) + model.save_pretrained_torchao = types.MethodType(unsloth_save_pretrained_torchao, model) pass return model pass