diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 69f36f0d46..3a29352a92 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2024.11.9" +__version__ = "2024.11.10" __all__ = [ "prepare_model_for_kbit_training", @@ -54,6 +54,7 @@ __all__ = [ "unpatch_gradient_checkpointing", "HAS_CUT_CROSS_ENTROPY", + "EMPTY_LOGITS", "fused_linear_cross_entropy", "patch_unsloth_smart_gradient_checkpointing", "unpatch_unsloth_smart_gradient_checkpointing", @@ -1128,14 +1129,27 @@ def unsloth_compile_transformers( debug = False, import_from_cache = False, disable = False, + return_logits = False, ): + if Version(torch_version) < Version("2.4.0"): + print( + "="*30 + \ + "Unsloth: Unfortunately Unsloth vision and other newer optimized models need Torch 2.4 or later.\n"\ + f"You have Torch version {torch_version}. Please upgrade your Torch version by visiting https://pytorch.org/\n"\ + "For now your models will not get optimized, but will still work for now!" + ) + return + pass + if disable: return + model_types = get_transformers_model_type( model_name = model_name, token = token, revision = revision, trust_remote_code = trust_remote_code, ) + for model_type in model_types: _unsloth_compile_transformers( model_type, @@ -1158,7 +1172,36 @@ def unsloth_compile_transformers( debug = debug, import_from_cache = import_from_cache, disable = disable, + return_logits = return_logits, ) pass return model_types pass + +# We need an empty logits flag to warn people logits will not be returned anymore unless asked ie +# os.environ['UNSLOTH_RETURN_LOGITS'] = '1' +LOGITS_ERROR_STRING = \ + "Unsloth: Logits are empty from 2024.11 onwards. To get raw logits again, please "\ + 'set the environment variable `UNSLOTH_RETURN_LOGITS` to `"1" BEFORE starting to train ie before `trainer.train()`. For example:\n\n'\ + "import os\n"\ + "os.environ['UNSLOTH_RETURN_LOGITS'] = '1'\n"\ + "... trainer.train() ..." + +def raise_logits_error(*args, **kwargs): raise NotImplementedError(LOGITS_ERROR_STRING) +def return_none(*args, **kwargs): return None +class EmptyLogits: + def __init__(self): return + def raise_getattr_error(self, attr): return return_none if attr == "to" else raise_logits_error + __getitem__ = raise_logits_error + __getattr__ = raise_getattr_error + def __repr__(self): return LOGITS_ERROR_STRING + def __str__ (self): return LOGITS_ERROR_STRING +pass +EMPTY_LOGITS = EmptyLogits() +functions = dir(torch.Tensor) +for j, function in enumerate(functions): + if function.startswith("__") and function.endswith("__"): + exec(f"def raise_{j}(*args, **kwargs): print('{function}')", globals(), locals()) + try: exec(f"EMPTY_LOGITS.{function} = raise_{j}", globals(), locals()) + except: continue +pass diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index 0256fc1830..bb5c841409 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -65,7 +65,7 @@ except: # Old HF Hub versions <= 0.0.25 from huggingface_hub.utils._token import get_token pass - +from triton import __version__ as triton_version def original_apply_qkv(self, X): Q = self.q_proj(X) @@ -980,7 +980,8 @@ def CausalLM_fast_forward(fast_forward_inference): elif num_logits_to_keep != 0: logits = self.lm_head(hidden_states[:, -num_logits_to_keep:, :].to(lm_head.dtype)) else: - if HAS_CUT_CROSS_ENTROPY and labels is not None: + RETURN_LOGITS = os.environ.get("UNSLOTH_RETURN_LOGITS", "0") == "1" + if not RETURN_LOGITS and HAS_CUT_CROSS_ENTROPY and labels is not None: n_items = kwargs.get("num_items_in_batch", None) or kwargs.get("n_items", None) loss = fused_linear_cross_entropy( hidden_states = hidden_states, @@ -993,13 +994,14 @@ def CausalLM_fast_forward(fast_forward_inference): output = (logits,) + outputs[1:] return (loss,) + output if loss is not None else output - return CausalLMOutputWithPast( + output = CausalLMOutputWithPast( loss=loss, - logits=None, + logits=EMPTY_LOGITS, past_key_values=outputs.past_key_values, hidden_states=outputs.hidden_states, attentions=outputs.attentions, ) + return output pass logits = self.lm_head(hidden_states.to(lm_head.dtype)) pass @@ -1547,9 +1549,9 @@ class FastLlamaModel: max_memory = round(gpu_stats.total_memory / 1024 / 1024 / 1024, 3) statistics = \ - f"==((====))== Unsloth {__version__}: Fast {model_patcher.__name__[4:-5]} patching. Transformers = {transformers_version}.\n"\ - f" \\\ /| GPU: {gpu_stats.name}. Max memory: {max_memory} GB. Platform = {platform_system}.\n"\ - f"O^O/ \_/ \\ Pytorch: {torch.__version__}. CUDA = {gpu_stats.major}.{gpu_stats.minor}. CUDA Toolkit = {torch.version.cuda}.\n"\ + f"==((====))== Unsloth {__version__}: Fast {model_patcher.__name__[4:-5]} patching. Transformers:{transformers_version}.\n"\ + f" \\\ /| GPU: {gpu_stats.name}. Max memory: {max_memory} GB. Platform: {platform_system}.\n"\ + f"O^O/ \_/ \\ Torch: {torch.__version__}. CUDA: {gpu_stats.major}.{gpu_stats.minor}. CUDA Toolkit: {torch.version.cuda}. Triton: {triton_version}\n"\ f"\ / Bfloat16 = {str(SUPPORTS_BFLOAT16).upper()}. FA [Xformers = {xformers_version}. FA2 = {HAS_FLASH_ATTENTION}]\n"\ f' "-____-" Free Apache license: http://github.com/unslothai/unsloth' print(statistics) diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index 232fe6acff..f8ed3a87e6 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -347,6 +347,7 @@ class FastVisionModel(FastBaseVisionModel): use_gradient_checkpointing = "unsloth", resize_model_vocab = None, # [TODO] No effect revision = None, + return_logits = False, # Return logits *args, **kwargs, ): if token is None: token = get_token() @@ -359,37 +360,11 @@ class FastVisionModel(FastBaseVisionModel): old_model_name = model_name model_name = get_model_name(model_name, load_in_4bit) - with contextlib.redirect_stdout(open(os.devnull, "w")): - patch_loss_functions(torch_compile = False) - model_types = unsloth_compile_transformers( - model_name = model_name, - sdpa_dynamic_mask = True, - sdpa_bool_masks = True, - sdpa_gqa_replace = True, - sdpa_dynamic_compile = True, - compile_attention = True, - disable_causal_masks = True, - compile_torch_modules = True, - compile_custom_modules = True, - compile_function_calls = True, - fuse_lm_head = True, - gradient_checkpointing = True, - manual_replacements = True, - epilogue_fusion = True, - max_autotune = False, - shape_padding = True, - cudagraphs = False, - debug = False, - import_from_cache = False, - disable = False, - ) - pass - # First check if it's a normal model via AutoConfig from huggingface_hub.utils import disable_progress_bars, enable_progress_bars, are_progress_bars_disabled was_disabled = are_progress_bars_disabled() disable_progress_bars() - + autoconfig_error = None peft_error = None try: @@ -473,6 +448,33 @@ class FastVisionModel(FastBaseVisionModel): if not was_disabled: enable_progress_bars() + with contextlib.redirect_stdout(open(os.devnull, "w")): + patch_loss_functions(torch_compile = False) + model_types = unsloth_compile_transformers( + model_name = model_name, + sdpa_dynamic_mask = True, + sdpa_bool_masks = True, + sdpa_gqa_replace = True, + sdpa_dynamic_compile = True, + compile_attention = True, + disable_causal_masks = True, + compile_torch_modules = True, + compile_custom_modules = True, + compile_function_calls = True, + fuse_lm_head = True, + gradient_checkpointing = True, + manual_replacements = True, + epilogue_fusion = True, + max_autotune = False, + shape_padding = True, + cudagraphs = False, + debug = False, + import_from_cache = False, + disable = False, + return_logits = return_logits, + ) + pass + # Check if this is local model since the tokenizer gets overwritten if os.path.exists(os.path.join(old_model_name, "tokenizer_config.json")) and \ os.path.exists(os.path.join(old_model_name, "tokenizer.json")) and \ diff --git a/unsloth/models/mapper.py b/unsloth/models/mapper.py index fc1dc8cdb0..b2f73aa6c2 100644 --- a/unsloth/models/mapper.py +++ b/unsloth/models/mapper.py @@ -492,6 +492,14 @@ __INT_TO_FLOAT_MAPPER = \ "unsloth/llava-v1.6-mistral-7b-hf", "llava-hf/llava-v1.6-mistral-7b-hf", ), + "unsloth/Llama-3.1-Tulu-3-8B-bnb-4bit" : ( + "unsloth/Llama-3.1-Tulu-3-8B", + "allenai/Llama-3.1-Tulu-3-8B", + ), + "unsloth/Llama-3.1-Tulu-3-70B-bnb-4bit" : ( + "unsloth/Llama-3.1-Tulu-3-70B", + "allenai/Llama-3.1-Tulu-3-70B", + ), } INT_TO_FLOAT_MAPPER = {} diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py index 69fb3fd986..80c1f82d4d 100644 --- a/unsloth/models/vision.py +++ b/unsloth/models/vision.py @@ -31,6 +31,7 @@ from unsloth_zoo.peft_utils import ( get_peft_regex, merge_and_overwrite_lora, ) +from triton import __version__ as triton_version __all__ = [ "FastBaseVisionModel", @@ -95,9 +96,9 @@ class FastBaseVisionModel: max_memory = round(gpu_stats.total_memory / 1024 / 1024 / 1024, 3) statistics = \ - f"==((====))== Unsloth {__version__}: Fast {model_types[0].title()} vision patching. Transformers = {transformers_version}.\n"\ - f" \\\ /| GPU: {gpu_stats.name}. Max memory: {max_memory} GB. Platform = {platform_system}.\n"\ - f"O^O/ \_/ \\ Pytorch: {torch.__version__}. CUDA = {gpu_stats.major}.{gpu_stats.minor}. CUDA Toolkit = {torch.version.cuda}.\n"\ + f"==((====))== Unsloth {__version__}: Fast {model_types[0].title()} vision patching. Transformers: {transformers_version}.\n"\ + f" \\\ /| GPU: {gpu_stats.name}. Max memory: {max_memory} GB. Platform: {platform_system}.\n"\ + f"O^O/ \_/ \\ Torch: {torch.__version__}. CUDA: {gpu_stats.major}.{gpu_stats.minor}. CUDA Toolkit: {torch.version.cuda}. Triton: {triton_version}\n"\ f"\ / Bfloat16 = {str(SUPPORTS_BFLOAT16).upper()}. FA [Xformers = {xformers_version}. FA2 = {HAS_FLASH_ATTENTION}]\n"\ f' "-____-" Free Apache license: http://github.com/unslothai/unsloth' print(statistics)