ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'main' of https://github.com/unslothai/unsloth

Browse source
This commit is contained in:
Daniel Han 2025-08-28 03:19:16 -07:00
commit eded24dad4
11 changed files with 52 additions and 41 deletions
Download patch file Download diff file Expand all files Collapse all files

4
unsloth/models/cohere.py
View file

@ -14,6 +14,8 @@
from .llama import *
from ._utils import __version__
from unsloth_zoo.hf_utils import dtype_from_config
from unsloth_zoo.utils import _get_dtype
try:
from transformers.models.cohere.modeling_cohere import (
CohereAttention,
@ -401,7 +403,7 @@ def CohereModel_fast_forward_inference(
out_weights = tuple(torch.empty_like(self.model.layers[0].input_layernorm.weight, dtype = torch.float32, device = torch.device(x)) for x in range(DEVICE_COUNT))
input_ids = input_ids[:,:self.max_seq_length]
hidden_states = self.model.embed_tokens(input_ids)
hidden_states = hidden_states.to(self.config.torch_dtype)
hidden_states = hidden_states.to(_get_dtype(dtype_from_config(self.config)))
bsz, q_len, hd = hidden_states.shape
seq_len = past_key_values[0][0].shape[-2]
if bsz != 1:

5
unsloth/models/falcon_h1.py
View file

@ -16,6 +16,7 @@ from .llama import *
import os
from ._utils import __version__
from unsloth_zoo.utils import Version, _get_dtype
from unsloth_zoo.hf_utils import dtype_from_config
from .llama import (
LlamaRotaryEmbedding,
LlamaLinearScalingRotaryEmbedding,
@ -480,7 +481,7 @@ def _FalconH1_fast_forward_inference(attention_fast_forward_inference=FalconH1At
X = self.model.embed_tokens(input_ids)
X = X * self.config.embedding_multiplier
X = X.to(_get_dtype(self.config.torch_dtype))
X = X.to(_get_dtype(dtype_from_config(self.config)))
bsz, q_len, hd = X.shape
assert(q_len == 1)
# Get saved buffers to reduce memory movement
@ -582,7 +583,7 @@ def _fast_prepare_inputs_for_generation(
position_ids=None,
use_cache=True,
**kwargs,):
# Overwitten -- has a unique cache type, `FalconHybridMambaAttentionDynamicCache`
# Overwritten -- has a unique cache type, `FalconHybridMambaAttentionDynamicCache`
empty_past_kv = past_key_values is None
# If we have cache: let's slice `input_ids` through `cache_position`, to keep only the unprocessed tokens

8
unsloth/models/gemma.py
View file

@ -14,6 +14,8 @@
from .llama import *
from ._utils import __version__
from unsloth_zoo.utils import _get_dtype
from unsloth_zoo.hf_utils import dtype_from_config
import math
try:
@ -152,7 +154,7 @@ def GemmaModel_fast_forward_inference(
out_weights = tuple(torch.empty_like(self.model.layers[0].input_layernorm.weight, dtype = torch.float32, device = torch.device(x)) for x in range(DEVICE_COUNT))
input_ids = input_ids[:,:self.max_seq_length]
hidden_states = self.model.embed_tokens(input_ids)
hidden_states = hidden_states.to(self.config.torch_dtype)
hidden_states = hidden_states.to(_get_dtype(dtype_from_config(self.config)))
# 3072**0.5 = 55.5000 in bfloat16, whilst 55.4256 in float32
# 2048**0.5 = 45.2500 in bfloat16, whilst 45.2548 in float32
hidden_states *= torch.tensor(math_sqrt(self.config.hidden_size), dtype = hidden_states.dtype)
@ -246,7 +248,7 @@ class GemmaFixedRotaryEmbedding(torch.nn.Module):
# in FP32. They are applied (multiplied) in FP32 as well.
self.current_rope_size = seq_len
# The difference is we do division explicity instead of t * (1/x) ie we do t/x.
# The difference is we do division explicitly instead of t * (1/x) ie we do t/x.
freq_exponents = (2.0 / self.dim) * (
torch.arange(self.dim // 2, dtype = torch.int64, device = "cpu").float()
)
@ -310,7 +312,7 @@ class GemmaFixedLinearScalingRotaryEmbedding(GemmaFixedRotaryEmbedding):
# in FP32. They are applied (multiplied) in FP32 as well.
self.current_rope_size = seq_len
# The difference is we do division explicity instead of t * (1/x) ie we do t/x.
# The difference is we do division explicitly instead of t * (1/x) ie we do t/x.
freq_exponents = (2.0 / self.dim) * (
torch.arange(self.dim // 2, dtype = torch.int64, device = "cpu").float()
)

4
unsloth/models/gemma2.py
View file

@ -14,6 +14,8 @@
from .llama import *
from ._utils import __version__
from unsloth_zoo.utils import _get_dtype
from unsloth_zoo.hf_utils import dtype_from_config
from .gemma import (
GemmaFixedRotaryEmbedding,
GemmaFixedLinearScalingRotaryEmbedding,
@ -379,7 +381,7 @@ def Gemma2Model_fast_forward_inference(
out_weights = tuple(torch.empty_like(self.model.layers[0].input_layernorm.weight, dtype = torch.float32, device = torch.device(x)) for x in range(DEVICE_COUNT))
input_ids = input_ids[:,:self.max_seq_length]
hidden_states = self.model.embed_tokens(input_ids)
hidden_states = hidden_states.to(self.config.torch_dtype)
hidden_states = hidden_states.to(_get_dtype(dtype_from_config(self.config)))
# 3072**0.5 = 55.5000 in bfloat16, whilst 55.4256 in float32
# 2048**0.5 = 45.2500 in bfloat16, whilst 45.2548 in float32
hidden_states *= torch.tensor(math_sqrt(self.config.hidden_size), dtype = hidden_states.dtype)

4
unsloth/models/granite.py
View file

@ -15,6 +15,8 @@
from .llama import *
import os
from ._utils import __version__
from unsloth_zoo.utils import _get_dtype
from unsloth_zoo.hf_utils import dtype_from_config
from .llama import (
LlamaRotaryEmbedding,
LlamaLinearScalingRotaryEmbedding,
@ -375,7 +377,7 @@ def GraniteModel_fast_forward_inference(
):
input_ids = input_ids[:,:self.max_seq_length]
hidden_states = self.model.embed_tokens(input_ids)
hidden_states = hidden_states.to(self.config.torch_dtype)
hidden_states = hidden_states.to(_get_dtype(dtype_from_config(self.config)))
hidden_states *= self.model.embedding_multiplier
residual_multiplier = \
self.residual_multiplier \

15
unsloth/models/llama.py
View file

@ -24,6 +24,7 @@ from ._utils import move_to_device
from torch.nn.functional import scaled_dot_product_attention
from transformers import __version__ as transformers_version
from unsloth_zoo.utils import Version, _get_dtype
from unsloth_zoo.hf_utils import dtype_from_config, add_dtype_kwargs
from unsloth_zoo.peft_utils import SKIP_QUANTIZATION_MODULES
from unsloth import DEVICE_TYPE, DEVICE_COUNT
@ -783,7 +784,7 @@ def LlamaModel_fast_forward(
if inputs_embeds is None:
inputs_embeds = self.embed_tokens(input_ids)
inputs_embeds = inputs_embeds.to(_get_dtype(self.config.torch_dtype))
inputs_embeds = inputs_embeds.to(_get_dtype(dtype_from_config(self.config)))
# Normalized from Gemma
IS_GEMMA = self.config.model_type.startswith("gemma")
@ -1057,7 +1058,7 @@ def _LlamaModel_fast_forward_inference(attention_fast_forward_inference=LlamaAtt
mlp_size = self.config.intermediate_size
X = self.model.embed_tokens(input_ids)
X = X.to(_get_dtype(self.config.torch_dtype))
X = X.to(_get_dtype(dtype_from_config(self.config)))
bsz, q_len, hd = X.shape
assert(q_len == 1)
# Get saved buffers to reduce memory movement
@ -1274,7 +1275,7 @@ def CausalLM_fast_forward(fast_forward_inference):
logits = self.lm_head(hidden_states.to(dtype))
pass
logits = logits.to(_get_dtype(self.config.torch_dtype))
logits = logits.to(_get_dtype(dtype_from_config(self.config)))
loss = None
logit_softcapping = getattr(self.config, "final_logit_softcapping", 0)
logit_scaling = getattr(self.config, "logit_scale", 0)
@ -1754,7 +1755,7 @@ def unsloth_fast_generate(
):
FastLlamaModel.for_inference(self)
dtype = _get_dtype(self.config.torch_dtype)
dtype = _get_dtype(dtype_from_config(self.config))
if hasattr(self, "config") and hasattr(self.config, "max_position_embeddings"):
if "input_ids" in kwargs and kwargs["input_ids"] is not None and "max_new_tokens" in kwargs:
@ -2023,12 +2024,14 @@ class FastLlamaModel:
# Cannot be None, since HF now checks for the config
if load_in_4bit: kwargs["quantization_config"] = bnb_config
kwargs = add_dtype_kwargs(dtype, kwargs)
raise_handler = RaiseUninitialized()
if num_labels is not None:
model = AutoModelForSequenceClassification.from_pretrained(
model_name,
device_map = device_map,
torch_dtype = dtype,
# torch_dtype = dtype, # transformers changed torch_dtype to dtype
num_labels = num_labels,
#quantization_config = bnb_config,
token = token,
@ -2041,7 +2044,7 @@ class FastLlamaModel:
model = AutoModelForCausalLM.from_pretrained(
model_name,
device_map = device_map,
torch_dtype = dtype,
# torch_dtype = dtype, # transformers changed torch_dtype to dtype
# quantization_config = bnb_config,
token = token,
max_position_embeddings = max_position_embeddings,

15
unsloth/models/loader.py
View file

@ -47,6 +47,7 @@ import importlib.util
# https://github.com/huggingface/transformers/pull/26037 allows 4 bit loading!
from unsloth_zoo.utils import Version, _get_dtype
from unsloth_zoo.hf_utils import dtype_from_config
transformers_version = Version(transformers_version)
SUPPORTS_FOURBIT = transformers_version >= Version("4.37")
SUPPORTS_GEMMA = transformers_version >= Version("4.38")
@ -437,12 +438,11 @@ class FastLanguageModel(FastLlamaModel):
if load_in_4bit:
# Fix up bitsandbytes config
config = model.config.to_dict()
torch_dtype = config.get("dtype") or config.get("torch_dtype")
compute_dtype = dtype_from_config(model.config)
quantization_config = \
{
# Sometimes torch_dtype is not a string!!
"bnb_4bit_compute_dtype" : torch_dtype,
# Sometimes compute_dtype is not a string!!
"bnb_4bit_compute_dtype" : compute_dtype,
"bnb_4bit_quant_type" : "nf4",
"bnb_4bit_use_double_quant" : True,
"llm_int8_enable_fp32_cpu_offload" : False,
@ -889,12 +889,11 @@ class FastModel(FastBaseModel):
if load_in_4bit:
# Fix up bitsandbytes config
config = model.config.to_dict()
torch_dtype = config.get("dtype") or config.get("torch_dtype")
compute_dtype = dtype_from_config(model.config)
quantization_config = \
{
# Sometimes torch_dtype is not a string!!
"bnb_4bit_compute_dtype" : torch_dtype,
# Sometimes compute_dtype is not a string!!
"bnb_4bit_compute_dtype" : compute_dtype,
"bnb_4bit_quant_type" : "nf4",
"bnb_4bit_use_double_quant" : True,
"llm_int8_enable_fp32_cpu_offload" : False,

6
unsloth/models/mistral.py
View file

@ -15,6 +15,8 @@
from .llama import *
import os
from ._utils import __version__
from unsloth_zoo.utils import _get_dtype
from unsloth_zoo.hf_utils import dtype_from_config
from .llama import (
LlamaRotaryEmbedding,
LlamaLinearScalingRotaryEmbedding,
@ -230,7 +232,7 @@ def MistralForCausalLM_fast_forward(
attention_mask = attention_mask.expand(bsz, 1, q_len, q_len)
attention_mask = attention_mask + causal_mask_values[None, None, :, :]
attention_mask = attention_mask.to(dtype=_get_dtype(self.config.torch_dtype))
attention_mask = attention_mask.to(dtype=_get_dtype(dtype_from_config(self.config)))
output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
output_hidden_states = (
@ -324,7 +326,7 @@ def MistralForCausalLM_fast_forward(
pass
logits = self.lm_head(hidden_states.to(lm_head.dtype))
pass
logits = logits.to(_get_dtype(self.config.torch_dtype))
logits = logits.to(_get_dtype(dtype_from_config(self.config)))
loss = None
if labels is not None:

4
unsloth/models/rl.py
View file

@ -234,7 +234,7 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"):
)
pass
# Edit bf16, fp16 by checking model's torch_dtype directly
# Edit bf16, fp16 by checking model's dtype/torch_dtype directly
extra_args = ""
if "args" in call_args and "model" in call_args:
mixed_precision = \
@ -247,7 +247,7 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"):
" print('Unsloth: Switching to float32 training since model cannot work with float16')\n"\
" force_float32 = True\n"\
"mixed_precision_dtype = os.environ.get('UNSLOTH_MIXED_PRECISION', 'float32')\n"\
"dtype = getattr(model.config, 'torch_dtype', None)\n"\
"dtype = getattr(model.config, 'dtype', None) or getattr(model.config, 'torch_dtype', None)\n"\
"if dtype is None: dtype = model.get_input_embeddings().dtype\n"\
"from unsloth_zoo.utils import _get_dtype\n"\
"dtype = _get_dtype(dtype)\n"\

15
unsloth/models/vision.py
View file

@ -43,6 +43,7 @@ from transformers.models.llama.modeling_llama import logger
from transformers import __version__ as transformers_version
from triton import __version__ as triton_version
from unsloth_zoo.utils import _get_dtype
from unsloth_zoo.hf_utils import dtype_from_config, add_dtype_kwargs
from unsloth_zoo.patching_utils import patch_model_and_tokenizer
from unsloth_zoo.training_utils import prepare_model_for_training
import types
@ -73,8 +74,6 @@ global PROMPT_LOOPKUP
PROMPT_LOOPKUP = dict()
from transformers import GenerationConfig, CompileConfig, HybridCache
from transformers import PretrainedConfig
HAS_TORCH_DTYPE = "torch_dtype" in PretrainedConfig.__doc__
_compile_config = CompileConfig(
fullgraph = False,
@ -121,7 +120,7 @@ def unsloth_base_fast_generate(
bsz = input_ids.shape[0]
FastBaseModel.for_inference(self)
dtype = _get_dtype(getattr(self.config, "dtype", None) or getattr(self.config, "torch_dtype", None))
dtype = _get_dtype(dtype_from_config(self.config))
# Check if VLM
is_vlm = any(
@ -444,11 +443,7 @@ class FastBaseModel:
torch_dtype = dtype
if do_forced_float32: torch_dtype = torch.bfloat16
if HAS_TORCH_DTYPE:
kwargs["torch_dtype"] = torch_dtype
else:
# Transformers removed torch_dtype
kwargs["dtype"] = torch_dtype
kwargs = add_dtype_kwargs(torch_dtype, kwargs)
raise_handler = RaiseUninitialized()
model = auto_model.from_pretrained(
@ -705,9 +700,7 @@ class FastBaseModel:
full_finetuning = os.environ.get("UNSLOTH_ENABLE_FULL_FINETUNING", "0") == "1"
float32_mixed_precision = True
if _get_dtype(
getattr(model.config, "dtype", None) or getattr(model.config, "torch_dtype", None)
) == torch.bfloat16 and full_finetuning:
if _get_dtype(dtype_from_config(model.config)) == torch.bfloat16 and full_finetuning:
# Use bfloat16 precision for full finetuning
float32_mixed_precision = False

13
unsloth/save.py
View file

@ -13,6 +13,7 @@
# limitations under the License.
from unsloth_zoo.utils import Version
from unsloth_zoo.hf_utils import dtype_from_config, HAS_TORCH_DTYPE
from bitsandbytes.nn import Linear4bit as Bnb_Linear4bit
from peft.tuners.lora import Linear4bit as Peft_Linear4bit
from peft.tuners.lora import Linear as Peft_Linear
@ -549,7 +550,7 @@ def unsloth_save_model(
from collections import OrderedDict
state_dict = OrderedDict()
torch_dtype = internal_model.config.torch_dtype
torch_dtype = dtype_from_config(internal_model.config)
if type(torch_dtype) is str:
if torch_dtype == "float16": torch_dtype = torch.float16
elif torch_dtype == "bfloat16": torch_dtype = torch.bfloat16
@ -1880,7 +1881,7 @@ def unsloth_save_pretrained_gguf(
for _ in range(3):
gc.collect()
model_dtype = self.config.torch_dtype
model_dtype = dtype_from_config(self.config)
model_type = self.config.model_type
if type(model_dtype) is str:
assert(model_dtype == "float16" or model_dtype == "bfloat16")
@ -2058,7 +2059,7 @@ def unsloth_push_to_hub_gguf(
for _ in range(3):
gc.collect()
model_dtype = self.config.torch_dtype
model_dtype = dtype_from_config(self.config)
model_type = self.config.model_type
if type(model_dtype) is str:
assert(model_dtype == "float16" or model_dtype == "bfloat16")
@ -2155,11 +2156,15 @@ def unsloth_save_pretrained_torchao(
quantization_config = TorchAoConfig(quant_type=torchao_config)
tokenizer = AutoTokenizer.from_pretrained(new_save_directory)
if HAS_TORCH_DTYPE:
kwargs = {"torch_dtype" : "auto"}
else:
kwargs = {"dtype" : "auto"}
model = AutoModelForCausalLM.from_pretrained(
new_save_directory,
torch_dtype="auto",
device_map="auto",
quantization_config=quantization_config,
**kwargs,
)
if push_to_hub: