From 16dea671cda55d22a4b9eef697d9fedeef518ce2 Mon Sep 17 00:00:00 2001 From: DefiWimar7 Date: Thu, 28 Aug 2025 10:44:28 +0800 Subject: [PATCH 1/2] chore: Fix Typos --- unsloth/models/falcon_h1.py | 2 +- unsloth/models/gemma.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/unsloth/models/falcon_h1.py b/unsloth/models/falcon_h1.py index 5643e3dfe9..83eb538f0c 100644 --- a/unsloth/models/falcon_h1.py +++ b/unsloth/models/falcon_h1.py @@ -582,7 +582,7 @@ def _fast_prepare_inputs_for_generation( position_ids=None, use_cache=True, **kwargs,): - # Overwitten -- has a unique cache type, `FalconHybridMambaAttentionDynamicCache` + # Overwritten -- has a unique cache type, `FalconHybridMambaAttentionDynamicCache` empty_past_kv = past_key_values is None # If we have cache: let's slice `input_ids` through `cache_position`, to keep only the unprocessed tokens diff --git a/unsloth/models/gemma.py b/unsloth/models/gemma.py index e43b205ecd..fc784be09d 100644 --- a/unsloth/models/gemma.py +++ b/unsloth/models/gemma.py @@ -246,7 +246,7 @@ class GemmaFixedRotaryEmbedding(torch.nn.Module): # in FP32. They are applied (multiplied) in FP32 as well. self.current_rope_size = seq_len - # The difference is we do division explicity instead of t * (1/x) ie we do t/x. + # The difference is we do division explicitly instead of t * (1/x) ie we do t/x. freq_exponents = (2.0 / self.dim) * ( torch.arange(self.dim // 2, dtype = torch.int64, device = "cpu").float() ) @@ -310,7 +310,7 @@ class GemmaFixedLinearScalingRotaryEmbedding(GemmaFixedRotaryEmbedding): # in FP32. They are applied (multiplied) in FP32 as well. self.current_rope_size = seq_len - # The difference is we do division explicity instead of t * (1/x) ie we do t/x. + # The difference is we do division explicitly instead of t * (1/x) ie we do t/x. freq_exponents = (2.0 / self.dim) * ( torch.arange(self.dim // 2, dtype = torch.int64, device = "cpu").float() ) From 603f52e1ffdf63c21bbfdac81e0c4bd7db305cb6 Mon Sep 17 00:00:00 2001 From: DoubleMathew Date: Thu, 28 Aug 2025 04:43:41 -0500 Subject: [PATCH 2/2] Handle transformers move to dtype from torch_dtype (#3225) --- unsloth/models/cohere.py | 4 +++- unsloth/models/falcon_h1.py | 3 ++- unsloth/models/gemma.py | 4 +++- unsloth/models/gemma2.py | 4 +++- unsloth/models/granite.py | 4 +++- unsloth/models/llama.py | 15 +++++++++------ unsloth/models/loader.py | 15 +++++++-------- unsloth/models/mistral.py | 6 ++++-- unsloth/models/rl.py | 4 ++-- unsloth/models/vision.py | 15 ++++----------- unsloth/save.py | 13 +++++++++---- 11 files changed, 49 insertions(+), 38 deletions(-) diff --git a/unsloth/models/cohere.py b/unsloth/models/cohere.py index d4691fb5d2..b22e5ca221 100644 --- a/unsloth/models/cohere.py +++ b/unsloth/models/cohere.py @@ -14,6 +14,8 @@ from .llama import * from ._utils import __version__ +from unsloth_zoo.hf_utils import dtype_from_config +from unsloth_zoo.utils import _get_dtype try: from transformers.models.cohere.modeling_cohere import ( CohereAttention, @@ -401,7 +403,7 @@ def CohereModel_fast_forward_inference( out_weights = tuple(torch.empty_like(self.model.layers[0].input_layernorm.weight, dtype = torch.float32, device = torch.device(x)) for x in range(DEVICE_COUNT)) input_ids = input_ids[:,:self.max_seq_length] hidden_states = self.model.embed_tokens(input_ids) - hidden_states = hidden_states.to(self.config.torch_dtype) + hidden_states = hidden_states.to(_get_dtype(dtype_from_config(self.config))) bsz, q_len, hd = hidden_states.shape seq_len = past_key_values[0][0].shape[-2] if bsz != 1: diff --git a/unsloth/models/falcon_h1.py b/unsloth/models/falcon_h1.py index 5643e3dfe9..dc0a9119dc 100644 --- a/unsloth/models/falcon_h1.py +++ b/unsloth/models/falcon_h1.py @@ -16,6 +16,7 @@ from .llama import * import os from ._utils import __version__ from unsloth_zoo.utils import Version, _get_dtype +from unsloth_zoo.hf_utils import dtype_from_config from .llama import ( LlamaRotaryEmbedding, LlamaLinearScalingRotaryEmbedding, @@ -480,7 +481,7 @@ def _FalconH1_fast_forward_inference(attention_fast_forward_inference=FalconH1At X = self.model.embed_tokens(input_ids) X = X * self.config.embedding_multiplier - X = X.to(_get_dtype(self.config.torch_dtype)) + X = X.to(_get_dtype(dtype_from_config(self.config))) bsz, q_len, hd = X.shape assert(q_len == 1) # Get saved buffers to reduce memory movement diff --git a/unsloth/models/gemma.py b/unsloth/models/gemma.py index e43b205ecd..0212911837 100644 --- a/unsloth/models/gemma.py +++ b/unsloth/models/gemma.py @@ -14,6 +14,8 @@ from .llama import * from ._utils import __version__ +from unsloth_zoo.utils import _get_dtype +from unsloth_zoo.hf_utils import dtype_from_config import math try: @@ -152,7 +154,7 @@ def GemmaModel_fast_forward_inference( out_weights = tuple(torch.empty_like(self.model.layers[0].input_layernorm.weight, dtype = torch.float32, device = torch.device(x)) for x in range(DEVICE_COUNT)) input_ids = input_ids[:,:self.max_seq_length] hidden_states = self.model.embed_tokens(input_ids) - hidden_states = hidden_states.to(self.config.torch_dtype) + hidden_states = hidden_states.to(_get_dtype(dtype_from_config(self.config))) # 3072**0.5 = 55.5000 in bfloat16, whilst 55.4256 in float32 # 2048**0.5 = 45.2500 in bfloat16, whilst 45.2548 in float32 hidden_states *= torch.tensor(math_sqrt(self.config.hidden_size), dtype = hidden_states.dtype) diff --git a/unsloth/models/gemma2.py b/unsloth/models/gemma2.py index 0f1924579a..43eefbbbfd 100644 --- a/unsloth/models/gemma2.py +++ b/unsloth/models/gemma2.py @@ -14,6 +14,8 @@ from .llama import * from ._utils import __version__ +from unsloth_zoo.utils import _get_dtype +from unsloth_zoo.hf_utils import dtype_from_config from .gemma import ( GemmaFixedRotaryEmbedding, GemmaFixedLinearScalingRotaryEmbedding, @@ -379,7 +381,7 @@ def Gemma2Model_fast_forward_inference( out_weights = tuple(torch.empty_like(self.model.layers[0].input_layernorm.weight, dtype = torch.float32, device = torch.device(x)) for x in range(DEVICE_COUNT)) input_ids = input_ids[:,:self.max_seq_length] hidden_states = self.model.embed_tokens(input_ids) - hidden_states = hidden_states.to(self.config.torch_dtype) + hidden_states = hidden_states.to(_get_dtype(dtype_from_config(self.config))) # 3072**0.5 = 55.5000 in bfloat16, whilst 55.4256 in float32 # 2048**0.5 = 45.2500 in bfloat16, whilst 45.2548 in float32 hidden_states *= torch.tensor(math_sqrt(self.config.hidden_size), dtype = hidden_states.dtype) diff --git a/unsloth/models/granite.py b/unsloth/models/granite.py index a3d79c8333..e07575900f 100644 --- a/unsloth/models/granite.py +++ b/unsloth/models/granite.py @@ -15,6 +15,8 @@ from .llama import * import os from ._utils import __version__ +from unsloth_zoo.utils import _get_dtype +from unsloth_zoo.hf_utils import dtype_from_config from .llama import ( LlamaRotaryEmbedding, LlamaLinearScalingRotaryEmbedding, @@ -375,7 +377,7 @@ def GraniteModel_fast_forward_inference( ): input_ids = input_ids[:,:self.max_seq_length] hidden_states = self.model.embed_tokens(input_ids) - hidden_states = hidden_states.to(self.config.torch_dtype) + hidden_states = hidden_states.to(_get_dtype(dtype_from_config(self.config))) hidden_states *= self.model.embedding_multiplier residual_multiplier = \ self.residual_multiplier \ diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index 231b17154d..6858517f25 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -24,6 +24,7 @@ from ._utils import move_to_device from torch.nn.functional import scaled_dot_product_attention from transformers import __version__ as transformers_version from unsloth_zoo.utils import Version, _get_dtype +from unsloth_zoo.hf_utils import dtype_from_config, add_dtype_kwargs from unsloth_zoo.peft_utils import SKIP_QUANTIZATION_MODULES from unsloth import DEVICE_TYPE, DEVICE_COUNT @@ -783,7 +784,7 @@ def LlamaModel_fast_forward( if inputs_embeds is None: inputs_embeds = self.embed_tokens(input_ids) - inputs_embeds = inputs_embeds.to(_get_dtype(self.config.torch_dtype)) + inputs_embeds = inputs_embeds.to(_get_dtype(dtype_from_config(self.config))) # Normalized from Gemma IS_GEMMA = self.config.model_type.startswith("gemma") @@ -1057,7 +1058,7 @@ def _LlamaModel_fast_forward_inference(attention_fast_forward_inference=LlamaAtt mlp_size = self.config.intermediate_size X = self.model.embed_tokens(input_ids) - X = X.to(_get_dtype(self.config.torch_dtype)) + X = X.to(_get_dtype(dtype_from_config(self.config))) bsz, q_len, hd = X.shape assert(q_len == 1) # Get saved buffers to reduce memory movement @@ -1274,7 +1275,7 @@ def CausalLM_fast_forward(fast_forward_inference): logits = self.lm_head(hidden_states.to(dtype)) pass - logits = logits.to(_get_dtype(self.config.torch_dtype)) + logits = logits.to(_get_dtype(dtype_from_config(self.config))) loss = None logit_softcapping = getattr(self.config, "final_logit_softcapping", 0) logit_scaling = getattr(self.config, "logit_scale", 0) @@ -1754,7 +1755,7 @@ def unsloth_fast_generate( ): FastLlamaModel.for_inference(self) - dtype = _get_dtype(self.config.torch_dtype) + dtype = _get_dtype(dtype_from_config(self.config)) if hasattr(self, "config") and hasattr(self.config, "max_position_embeddings"): if "input_ids" in kwargs and kwargs["input_ids"] is not None and "max_new_tokens" in kwargs: @@ -2023,12 +2024,14 @@ class FastLlamaModel: # Cannot be None, since HF now checks for the config if load_in_4bit: kwargs["quantization_config"] = bnb_config + kwargs = add_dtype_kwargs(dtype, kwargs) + raise_handler = RaiseUninitialized() if num_labels is not None: model = AutoModelForSequenceClassification.from_pretrained( model_name, device_map = device_map, - torch_dtype = dtype, + # torch_dtype = dtype, # transformers changed torch_dtype to dtype num_labels = num_labels, #quantization_config = bnb_config, token = token, @@ -2041,7 +2044,7 @@ class FastLlamaModel: model = AutoModelForCausalLM.from_pretrained( model_name, device_map = device_map, - torch_dtype = dtype, + # torch_dtype = dtype, # transformers changed torch_dtype to dtype # quantization_config = bnb_config, token = token, max_position_embeddings = max_position_embeddings, diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index 7710aba3c9..38d873f3df 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -47,6 +47,7 @@ import importlib.util # https://github.com/huggingface/transformers/pull/26037 allows 4 bit loading! from unsloth_zoo.utils import Version, _get_dtype +from unsloth_zoo.hf_utils import dtype_from_config transformers_version = Version(transformers_version) SUPPORTS_FOURBIT = transformers_version >= Version("4.37") SUPPORTS_GEMMA = transformers_version >= Version("4.38") @@ -437,12 +438,11 @@ class FastLanguageModel(FastLlamaModel): if load_in_4bit: # Fix up bitsandbytes config - config = model.config.to_dict() - torch_dtype = config.get("dtype") or config.get("torch_dtype") + compute_dtype = dtype_from_config(model.config) quantization_config = \ { - # Sometimes torch_dtype is not a string!! - "bnb_4bit_compute_dtype" : torch_dtype, + # Sometimes compute_dtype is not a string!! + "bnb_4bit_compute_dtype" : compute_dtype, "bnb_4bit_quant_type" : "nf4", "bnb_4bit_use_double_quant" : True, "llm_int8_enable_fp32_cpu_offload" : False, @@ -889,12 +889,11 @@ class FastModel(FastBaseModel): if load_in_4bit: # Fix up bitsandbytes config - config = model.config.to_dict() - torch_dtype = config.get("dtype") or config.get("torch_dtype") + compute_dtype = dtype_from_config(model.config) quantization_config = \ { - # Sometimes torch_dtype is not a string!! - "bnb_4bit_compute_dtype" : torch_dtype, + # Sometimes compute_dtype is not a string!! + "bnb_4bit_compute_dtype" : compute_dtype, "bnb_4bit_quant_type" : "nf4", "bnb_4bit_use_double_quant" : True, "llm_int8_enable_fp32_cpu_offload" : False, diff --git a/unsloth/models/mistral.py b/unsloth/models/mistral.py index ef046d1f3c..16a23f8853 100644 --- a/unsloth/models/mistral.py +++ b/unsloth/models/mistral.py @@ -15,6 +15,8 @@ from .llama import * import os from ._utils import __version__ +from unsloth_zoo.utils import _get_dtype +from unsloth_zoo.hf_utils import dtype_from_config from .llama import ( LlamaRotaryEmbedding, LlamaLinearScalingRotaryEmbedding, @@ -230,7 +232,7 @@ def MistralForCausalLM_fast_forward( attention_mask = attention_mask.expand(bsz, 1, q_len, q_len) attention_mask = attention_mask + causal_mask_values[None, None, :, :] - attention_mask = attention_mask.to(dtype=_get_dtype(self.config.torch_dtype)) + attention_mask = attention_mask.to(dtype=_get_dtype(dtype_from_config(self.config))) output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( @@ -324,7 +326,7 @@ def MistralForCausalLM_fast_forward( pass logits = self.lm_head(hidden_states.to(lm_head.dtype)) pass - logits = logits.to(_get_dtype(self.config.torch_dtype)) + logits = logits.to(_get_dtype(dtype_from_config(self.config))) loss = None if labels is not None: diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py index 19215fbf6b..ff0e04ee4f 100644 --- a/unsloth/models/rl.py +++ b/unsloth/models/rl.py @@ -234,7 +234,7 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"): ) pass - # Edit bf16, fp16 by checking model's torch_dtype directly + # Edit bf16, fp16 by checking model's dtype/torch_dtype directly extra_args = "" if "args" in call_args and "model" in call_args: mixed_precision = \ @@ -247,7 +247,7 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"): " print('Unsloth: Switching to float32 training since model cannot work with float16')\n"\ " force_float32 = True\n"\ "mixed_precision_dtype = os.environ.get('UNSLOTH_MIXED_PRECISION', 'float32')\n"\ - "dtype = getattr(model.config, 'torch_dtype', None)\n"\ + "dtype = getattr(model.config, 'dtype', None) or getattr(model.config, 'torch_dtype', None)\n"\ "if dtype is None: dtype = model.get_input_embeddings().dtype\n"\ "from unsloth_zoo.utils import _get_dtype\n"\ "dtype = _get_dtype(dtype)\n"\ diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py index fc31032594..debf43c1a9 100644 --- a/unsloth/models/vision.py +++ b/unsloth/models/vision.py @@ -43,6 +43,7 @@ from transformers.models.llama.modeling_llama import logger from transformers import __version__ as transformers_version from triton import __version__ as triton_version from unsloth_zoo.utils import _get_dtype +from unsloth_zoo.hf_utils import dtype_from_config, add_dtype_kwargs from unsloth_zoo.patching_utils import patch_model_and_tokenizer from unsloth_zoo.training_utils import prepare_model_for_training import types @@ -73,8 +74,6 @@ global PROMPT_LOOPKUP PROMPT_LOOPKUP = dict() from transformers import GenerationConfig, CompileConfig, HybridCache -from transformers import PretrainedConfig -HAS_TORCH_DTYPE = "torch_dtype" in PretrainedConfig.__doc__ _compile_config = CompileConfig( fullgraph = False, @@ -121,7 +120,7 @@ def unsloth_base_fast_generate( bsz = input_ids.shape[0] FastBaseModel.for_inference(self) - dtype = _get_dtype(getattr(self.config, "dtype", None) or getattr(self.config, "torch_dtype", None)) + dtype = _get_dtype(dtype_from_config(self.config)) # Check if VLM is_vlm = any( @@ -444,11 +443,7 @@ class FastBaseModel: torch_dtype = dtype if do_forced_float32: torch_dtype = torch.bfloat16 - if HAS_TORCH_DTYPE: - kwargs["torch_dtype"] = torch_dtype - else: - # Transformers removed torch_dtype - kwargs["dtype"] = torch_dtype + kwargs = add_dtype_kwargs(torch_dtype, kwargs) raise_handler = RaiseUninitialized() model = auto_model.from_pretrained( @@ -705,9 +700,7 @@ class FastBaseModel: full_finetuning = os.environ.get("UNSLOTH_ENABLE_FULL_FINETUNING", "0") == "1" float32_mixed_precision = True - if _get_dtype( - getattr(model.config, "dtype", None) or getattr(model.config, "torch_dtype", None) - ) == torch.bfloat16 and full_finetuning: + if _get_dtype(dtype_from_config(model.config)) == torch.bfloat16 and full_finetuning: # Use bfloat16 precision for full finetuning float32_mixed_precision = False diff --git a/unsloth/save.py b/unsloth/save.py index 4e7c7c6755..eace358cbb 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -13,6 +13,7 @@ # limitations under the License. from unsloth_zoo.utils import Version +from unsloth_zoo.hf_utils import dtype_from_config, HAS_TORCH_DTYPE from bitsandbytes.nn import Linear4bit as Bnb_Linear4bit from peft.tuners.lora import Linear4bit as Peft_Linear4bit from peft.tuners.lora import Linear as Peft_Linear @@ -549,7 +550,7 @@ def unsloth_save_model( from collections import OrderedDict state_dict = OrderedDict() - torch_dtype = internal_model.config.torch_dtype + torch_dtype = dtype_from_config(internal_model.config) if type(torch_dtype) is str: if torch_dtype == "float16": torch_dtype = torch.float16 elif torch_dtype == "bfloat16": torch_dtype = torch.bfloat16 @@ -1880,7 +1881,7 @@ def unsloth_save_pretrained_gguf( for _ in range(3): gc.collect() - model_dtype = self.config.torch_dtype + model_dtype = dtype_from_config(self.config) model_type = self.config.model_type if type(model_dtype) is str: assert(model_dtype == "float16" or model_dtype == "bfloat16") @@ -2058,7 +2059,7 @@ def unsloth_push_to_hub_gguf( for _ in range(3): gc.collect() - model_dtype = self.config.torch_dtype + model_dtype = dtype_from_config(self.config) model_type = self.config.model_type if type(model_dtype) is str: assert(model_dtype == "float16" or model_dtype == "bfloat16") @@ -2155,11 +2156,15 @@ def unsloth_save_pretrained_torchao( quantization_config = TorchAoConfig(quant_type=torchao_config) tokenizer = AutoTokenizer.from_pretrained(new_save_directory) + if HAS_TORCH_DTYPE: + kwargs = {"torch_dtype" : "auto"} + else: + kwargs = {"dtype" : "auto"} model = AutoModelForCausalLM.from_pretrained( new_save_directory, - torch_dtype="auto", device_map="auto", quantization_config=quantization_config, + **kwargs, ) if push_to_hub: