From c505a060caad153dcf039600702a24041b4131f0 Mon Sep 17 00:00:00 2001 From: yash solanki Date: Sun, 10 Aug 2025 01:41:00 +0530 Subject: [PATCH 1/4] feat(phi2): add FastPhiModel with partial RoPE and deterministic dropout; wire loader dispatch, alias mapping, and kernel hooks; add Phi-2 smoke test --- .../test_unsloth_qlora_train_and_merge.py | 25 ++ unsloth/kernels/__init__.py | 3 + unsloth/kernels/dropout.py | 70 ++++ unsloth/kernels/gelu.py | 38 ++ unsloth/kernels/layernorm.py | 15 + unsloth/kernels/rope_embedding.py | 40 ++ unsloth/models/__init__.py | 1 + unsloth/models/loader.py | 9 + unsloth/models/mapper.py | 4 + unsloth/models/phi.py | 355 ++++++++++++++++++ 10 files changed, 560 insertions(+) create mode 100644 unsloth/kernels/dropout.py create mode 100644 unsloth/kernels/gelu.py create mode 100644 unsloth/models/phi.py diff --git a/tests/qlora/test_unsloth_qlora_train_and_merge.py b/tests/qlora/test_unsloth_qlora_train_and_merge.py index 9040ad793d..953aefcca7 100644 --- a/tests/qlora/test_unsloth_qlora_train_and_merge.py +++ b/tests/qlora/test_unsloth_qlora_train_and_merge.py @@ -209,3 +209,28 @@ if __name__ == "__main__": ) with header_footer_context("Responses after unsloth merge to 16bit"): check_responses(responses, answer = ANSWER, prompt = prompt) + + +# Minimal Phi-2 smoke test to ensure loader + forward path works. +# Skips automatically if model cannot be downloaded in CI. +def test_unsloth_phi2_load_and_forward_smoke(): + import pytest + import torch + from unsloth import FastLanguageModel + + model_name = "microsoft/Phi-2" + try: + model, tokenizer = FastLanguageModel.from_pretrained( + model_name, + max_seq_length=64, + load_in_4bit=True, + use_exact_model_name=True, + ) + except Exception as e: + pytest.skip(f"Skipping Phi-2 smoke test due to: {e}") + + model.eval() + with torch.no_grad(): + input_ids = tokenizer("Hello", return_tensors="pt").input_ids.to(next(model.parameters()).device) + out = model(input_ids=input_ids) + assert hasattr(out, "logits") diff --git a/unsloth/kernels/__init__.py b/unsloth/kernels/__init__.py index 15913413d9..5b906b6cfa 100644 --- a/unsloth/kernels/__init__.py +++ b/unsloth/kernels/__init__.py @@ -25,8 +25,11 @@ from .rms_layernorm import ( from .layernorm import ( fast_layernorm, patch_layernorm, + fast_layernorm_inference, ) from .rope_embedding import fast_rope_embedding, inplace_rope_embedding +from .dropout import DeterministicDropout, seeded_dropout +from .gelu import fast_gelu, FastGELU from .swiglu import swiglu_fg_kernel, swiglu_DWf_DW_dfg_kernel from .geglu import ( geglu_exact_forward_kernel, diff --git a/unsloth/kernels/dropout.py b/unsloth/kernels/dropout.py new file mode 100644 index 0000000000..66563ce1e8 --- /dev/null +++ b/unsloth/kernels/dropout.py @@ -0,0 +1,70 @@ +# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import torch + +_UINT32_MAX_INV = 1.0 / 4294967295.0 + + +def _mix32(x: torch.Tensor) -> torch.Tensor: + x = (x + 0x9E3779B9) & 0xFFFFFFFF + x ^= (x >> 16) + x = (x * 0x85EBCA6B) & 0xFFFFFFFF + x ^= (x >> 13) + x = (x * 0xC2B2AE35) & 0xFFFFFFFF + x ^= (x >> 16) + return x + + +@torch.compiler.disable +def seeded_dropout(x: torch.Tensor, p: float, seed: int, scale: bool = True) -> torch.Tensor: + if p <= 0.0 or not (x.requires_grad or x.training if hasattr(x, 'training') else True): + return x + device = x.device + dtype = x.dtype + bsz, seqlen, hidden = x.shape[0], x.shape[1], x.shape[-1] + # Indices grids (broadcasted), keep memory modest by composing increments + b_idx = torch.arange(bsz, device=device, dtype=torch.int64).view(bsz, 1, 1) + t_idx = torch.arange(seqlen, device=device, dtype=torch.int64).view(1, seqlen, 1) + c_idx = torch.arange(hidden, device=device, dtype=torch.int64).view(1, 1, hidden) + + # Large coprime-like multipliers for mixing + mixed = (b_idx * 0x1F123BB5 + t_idx * 0x5DEECE66D + c_idx * 0xB5297A4D + (seed & 0xFFFFFFFF)) & 0xFFFFFFFF + rnd = _mix32(mixed).to(torch.float32) * _UINT32_MAX_INV + mask = (rnd >= p).to(dtype) + if scale and p < 1.0: + mask = mask / (1.0 - p) + return x * mask + + +class DeterministicDropout(torch.nn.Module): + def __init__(self, p: float, seed: int = 3407): + super().__init__() + self.p = float(p) + # Allow override via env variable + env_seed = os.environ.get("UNSLOTH_DROPOUT_SEED", None) + self.seed = int(env_seed) if env_seed is not None else int(seed) + self.register_buffer('_counter', torch.zeros((), dtype=torch.int64), persistent=False) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + if (not self.training) or self.p <= 0.0: + return x + # Derive a new seed per call to decorrelate successive uses + local = int(self._counter.item()) + self._counter.add_(1) + derived_seed = (self.seed + 0x9E3779B9 * local) & 0xFFFFFFFF + return seeded_dropout(x, self.p, derived_seed, scale=True) + + diff --git a/unsloth/kernels/gelu.py b/unsloth/kernels/gelu.py new file mode 100644 index 0000000000..c3b0499c3b --- /dev/null +++ b/unsloth/kernels/gelu.py @@ -0,0 +1,38 @@ +# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +import torch.nn.functional as F + + +@torch.compiler.disable +def fast_gelu(x: torch.Tensor, approximate: str | None = None) -> torch.Tensor: + """Fast GeLU wrapper. Uses torch.nn.functional.gelu with optional approximation. + + approximate: None | "tanh" + """ + if approximate is None: + return F.gelu(x) + return F.gelu(x, approximate=approximate) + + +class FastGELU(torch.nn.Module): + def __init__(self, approximate: str | None = None): + super().__init__() + self.approximate = approximate + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return fast_gelu(x, self.approximate) + + diff --git a/unsloth/kernels/layernorm.py b/unsloth/kernels/layernorm.py index 9e64c3d341..a09a541aff 100644 --- a/unsloth/kernels/layernorm.py +++ b/unsloth/kernels/layernorm.py @@ -181,6 +181,21 @@ def fast_layernorm(layernorm, X): out = Fast_Layernorm.apply(X, W, bias, eps) return out +# Public helper mirroring RMSNorm API for standard LayerNorm cases +@torch.compiler.disable +def fast_layernorm_inference(layernorm, X: torch.Tensor, out_weight: torch.Tensor | None = None): + XX = X.to(torch.float32, copy=True) + mean = XX.mean(-1, keepdim=True) + XX -= mean + var = (XX * XX).mean(-1, keepdim=True) + var += layernorm.eps if hasattr(layernorm, "eps") else layernorm.variance_epsilon + XX *= var.rsqrt_() + if out_weight is None: + return (XX * layernorm.weight).to(X.dtype) + out_weight[:] = layernorm.weight + return (XX * out_weight).to(X.dtype) + + def test_layernorm( dim = 1024, diff --git a/unsloth/kernels/rope_embedding.py b/unsloth/kernels/rope_embedding.py index fcc9cb923b..f1b6c363ae 100644 --- a/unsloth/kernels/rope_embedding.py +++ b/unsloth/kernels/rope_embedding.py @@ -463,3 +463,43 @@ def inplace_rope_embedding(Q, K, cos, sin, position_ids): K = Slow_RoPE_Embedding.apply(K, cos, sin, position_ids) torch_device_stream(Q.device).synchronize() return Q, K +pass + + +@torch.compiler.disable +def fast_partial_rope_embedding(Q, K, cos, sin, rotary_dim: int): + """Apply RoPE only to the first rotary_dim features of Q and K using the fast kernel. + + Shapes: + Q, K: [bsz, n_heads, seqlen, head_dim] + cos, sin: broadcastable to [seqlen, rotary_dim] + """ + if rotary_dim <= 0: + return Q, K + Q_rot = Q[..., :rotary_dim] + K_rot = K[..., :rotary_dim] + Q_rot2 = Fast_RoPE_Embedding.apply(Q_rot.transpose(1, 2), cos, sin).transpose(1, 2) + K_rot2 = Fast_RoPE_Embedding.apply(K_rot.transpose(1, 2), cos, sin).transpose(1, 2) + Q[..., :rotary_dim] = Q_rot2 + K[..., :rotary_dim] = K_rot2 + return Q, K +pass + + +def inplace_partial_rope_embedding(Q, K, cos, sin, position_ids, rotary_dim: int): + """Apply RoPE only to the first rotary_dim features of Q and K using the slow kernel. + + Shapes: + Q, K: [bsz, n_heads, seqlen, head_dim] + cos, sin: broadcastable to [seqlen, rotary_dim] + """ + if rotary_dim <= 0: + return Q, K + Q_rot = Q[..., :rotary_dim] + K_rot = K[..., :rotary_dim] + Q_rot2 = Slow_RoPE_Embedding.apply(Q_rot, cos, sin, position_ids) + K_rot2 = Slow_RoPE_Embedding.apply(K_rot, cos, sin, position_ids) + Q[..., :rotary_dim] = Q_rot2 + K[..., :rotary_dim] = K_rot2 + return Q, K +pass diff --git a/unsloth/models/__init__.py b/unsloth/models/__init__.py index 138f309032..7b31d9fbc6 100644 --- a/unsloth/models/__init__.py +++ b/unsloth/models/__init__.py @@ -20,6 +20,7 @@ from .qwen3 import FastQwen3Model from .qwen3_moe import FastQwen3MoeModel from .granite import FastGraniteModel from .sentence_transformer import FastSentenceTransformer +from .phi import FastPhiModel try: from .falcon_h1 import FastFalconH1Model diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index bd15ed5281..e58259846d 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -29,6 +29,7 @@ from .qwen2 import FastQwen2Model from .qwen3 import FastQwen3Model from .qwen3_moe import FastQwen3MoeModel from .cohere import FastCohereModel +from .phi import FastPhiModel from transformers import AutoConfig from transformers import __version__ as transformers_version from peft import PeftConfig, PeftModel @@ -635,6 +636,8 @@ class FastLanguageModel(FastLlamaModel): # f'Try `pip install --upgrade "transformers>=4.50.3"`\n'\ # f"to obtain the latest transformers build, then restart this session."\ # ) + elif model_type == "phi": + dispatch_model = FastPhiModel # Temporary disable optimized Cohere until errors match # elif model_type == "cohere": # dispatch_model = FastCohereModel @@ -742,6 +745,12 @@ class FastLanguageModel(FastLlamaModel): ] ) + # Allow model-specific post patches (e.g., Phi-2 defaults) + try: + model, tokenizer = dispatch_model.post_patch(model, tokenizer) + except Exception: + pass + if load_in_4bit: # Fix up bitsandbytes config, but respect user-provided quantization_config if quantization_config is None: diff --git a/unsloth/models/mapper.py b/unsloth/models/mapper.py index f0f430eb7e..e2a8944035 100644 --- a/unsloth/models/mapper.py +++ b/unsloth/models/mapper.py @@ -609,6 +609,10 @@ __INT_TO_FLOAT_MAPPER = \ "microsoft/phi-4", "unsloth/phi-4-bnb-4bit", ), + "unsloth/Phi-2-bnb-4bit" : ( + "unsloth/Phi-2", + "microsoft/Phi-2", + ), "unsloth/DeepSeek-R1-Distill-Qwen-32B-bnb-4bit" : ( "unsloth/DeepSeek-R1-Distill-Qwen-32B", "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B", diff --git a/unsloth/models/phi.py b/unsloth/models/phi.py new file mode 100644 index 0000000000..0ff46cb2b1 --- /dev/null +++ b/unsloth/models/phi.py @@ -0,0 +1,355 @@ +# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .llama import * +from ._utils import __version__ +from unsloth_zoo.utils import Version, _get_dtype +from .vision import FastBaseModel +from ..kernels import DeterministicDropout + +import torch +from typing import Optional, Tuple + +try: + from transformers import __version__ as transformers_version + transformers_version = Version(transformers_version) + from transformers.models.phi.modeling_phi import ( + PhiAttention, + PhiDecoderLayer, + PhiModel, + PhiForCausalLM, + ) + try: + from transformers.models.phi.modeling_phi import PhiSdpaAttention, PhiFlashAttention2 + except Exception: + PhiSdpaAttention = PhiAttention + PhiFlashAttention2 = PhiAttention +except Exception as error: + # We only import when actually used; loader will guard by AutoConfig + PhiAttention = None + PhiDecoderLayer = None + PhiModel = None + PhiForCausalLM = None + PhiSdpaAttention = None + PhiFlashAttention2 = None + + +def _phi_get_rotary_dims(attn_module) -> int: + head_dim: int = attn_module.head_dim + # Prefer explicit rotary_dim if provided by config + rotary_dim = getattr(attn_module.config, "rotary_dim", None) + if isinstance(rotary_dim, int) and 0 < rotary_dim <= head_dim: + # Ensure even for half-rotate math + return (rotary_dim // 2) * 2 + # Else use partial_rotary_factor if present + fraction = getattr(attn_module.config, "partial_rotary_factor", None) + if isinstance(fraction, (float, int)) and 0 < fraction <= 1: + rotary_dim = int(head_dim * float(fraction)) + return (rotary_dim // 2) * 2 + # Default: full rotation + return (head_dim // 2) * 2 if head_dim % 2 != 0 else head_dim + + +def PhiAttention_fast_forward( + self, + hidden_states: torch.Tensor, + causal_mask: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Tuple[torch.Tensor]] = None, + output_attentions: bool = False, + use_cache: bool = False, + padding_mask: Optional[torch.LongTensor] = None, + position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + *args, **kwargs, +) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + + # Clear inference caches if any (mirrors other model fastpaths) + if hasattr(self, "paged_attention"): + del self.paged_attention_K + del self.paged_attention_V + del self.paged_attention + del self.temp_QA + del self.temp_KV + del self.RH_Q + del self.attention + pass + + bsz, q_len, _ = hidden_states.size() + + n_heads: int = self.config.num_attention_heads + n_kv_heads: int = getattr(self.config, "num_key_value_heads", n_heads) + n_groups_attr = getattr(self, "num_key_value_groups", None) + n_groups: int = n_groups_attr if isinstance(n_groups_attr, int) and n_groups_attr > 0 else max(1, n_heads // max(1, n_kv_heads)) + head_dim: int = self.head_dim + assert (n_kv_heads * n_groups == n_heads) + + # Q, K, V projections + Q, K, V = self.apply_qkv(self, hidden_states) + Q = Q.view(bsz, q_len, n_heads, head_dim).transpose(1, 2) + K = K.view(bsz, q_len, n_kv_heads, head_dim).transpose(1, 2) + V = V.view(bsz, q_len, n_kv_heads, head_dim).transpose(1, 2) + + # Sequence lengths with KV cache + kv_seq_len = K.shape[-2] + if past_key_value is not None: + kv_seq_len += past_key_value[0].shape[-2] + + # Apply (partial) RoPE on first rotary_dim dims of Q, K + rotary_dim: int = _phi_get_rotary_dims(self) + if position_embeddings is not None: + cos, sin = position_embeddings + if rotary_dim < head_dim: + cos = cos[..., :rotary_dim] + sin = sin[..., :rotary_dim] + # Fast path when position_ids provided handled below via inplace op + if position_ids is None: + Q_rot = Q[..., :rotary_dim] + K_rot = K[..., :rotary_dim] + Q_rot, K_rot = inplace_rope_embedding(Q_rot, K_rot, cos, sin, position_ids) + Q[..., :rotary_dim] = Q_rot + K[..., :rotary_dim] = K_rot + else: + Q_rot = Q[..., :rotary_dim] + K_rot = K[..., :rotary_dim] + cos = cos[position_ids].unsqueeze(1) # [bs, 1, seqlen, dim] + sin = sin[position_ids].unsqueeze(1) + Q_rot, K_rot = inplace_rope_embedding(Q_rot, K_rot, cos, sin, position_ids) + Q[..., :rotary_dim] = Q_rot + K[..., :rotary_dim] = K_rot + else: + # Compute cos/sin from available rotary embedding; if none, create a local one + rope_module = None + if hasattr(self, "rotary_emb"): + rope_module = self.rotary_emb + rope_module.extend_rope_embedding(V, seq_len=kv_seq_len) + if position_ids is None: + cos = rope_module.cos_cached + sin = rope_module.sin_cached + else: + cos, sin = rope_module(V, seq_len=kv_seq_len) + else: + rope_module = getattr(self, "_unsloth_phi_rope", None) + if rope_module is None: + # Build Llama-style rotary embedding configured for Phi + try: + base = getattr(self.config, "rope_theta", 10000) + max_pos = getattr(self.config, "max_position_embeddings", 2048) + except Exception: + base = 10000 + max_pos = 2048 + rope_module = LlamaRotaryEmbedding(dim=head_dim, max_position_embeddings=max_pos, base=base, device=V.device) + # Keep for reuse + self._unsloth_phi_rope = rope_module + # Ensure buffers sized appropriately + rope_module.extend_rope_embedding(V, seq_len=kv_seq_len) + if position_ids is None: + cos = rope_module.cos_cached + sin = rope_module.sin_cached + else: + cos, sin = rope_module(V, seq_len=kv_seq_len) + + # Apply (partial) RoPE + if rotary_dim < head_dim: + cos = cos[..., :rotary_dim] + sin = sin[..., :rotary_dim] + Q_rot = Q[..., :rotary_dim] + K_rot = K[..., :rotary_dim] + Q_rot, K_rot = inplace_rope_embedding(Q_rot, K_rot, cos, sin, position_ids) + Q[..., :rotary_dim] = Q_rot + K[..., :rotary_dim] = K_rot + + # KV cache update + if past_key_value is not None: + K = torch.cat([past_key_value[0], K], dim=2) + V = torch.cat([past_key_value[1], V], dim=2) + past_key_value = (K, V) if use_cache else None + + # Attention computation (dispatch as in other models) + if (not HAS_FLASH_ATTENTION and HAS_XFORMERS and attention_mask is None): + # Xformers memory efficient attention with (bsz, seqlen, heads, dim) + Q = Q.transpose(1, 2) + K = K.transpose(1, 2) + V = V.transpose(1, 2) + + # Grouped Query Attention (expand KV across groups) + if n_groups != 1: + K = K.view(bsz, kv_seq_len, n_kv_heads, 1, head_dim) + V = V.view(bsz, kv_seq_len, n_kv_heads, 1, head_dim) + K = K.expand(bsz, kv_seq_len, n_kv_heads, n_groups, head_dim) + V = V.expand(bsz, kv_seq_len, n_kv_heads, n_groups, head_dim) + if hidden_states.requires_grad: + K = K.reshape(bsz, kv_seq_len, n_heads, head_dim) + V = V.reshape(bsz, kv_seq_len, n_heads, head_dim) + else: + Q = Q.view(bsz, q_len, n_kv_heads, n_groups, head_dim) + A = xformers_attention(Q, K, V, attn_bias=causal_mask) + A = A.view(bsz, q_len, n_heads, head_dim) + + elif HAS_FLASH_ATTENTION and attention_mask is None: + Q = Q.transpose(1, 2) + K = K.transpose(1, 2) + V = V.transpose(1, 2) + A = flash_attn_func(Q, K, V, causal=True) + else: + # SDPA fallback, support GQA if available + if SDPA_HAS_GQA: + A = scaled_dot_product_attention(Q, K, V, attn_mask=attention_mask, is_causal=False, enable_gqa=n_groups != 1) + A = A.transpose(1, 2) + else: + if n_groups != 1: + K = K[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, kv_seq_len, head_dim) + V = V[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, kv_seq_len, head_dim) + K = K.reshape(bsz, n_heads, kv_seq_len, head_dim) + V = V.reshape(bsz, n_heads, kv_seq_len, head_dim) + Q, K, V = Q.contiguous(), K.contiguous(), V.contiguous() + A = scaled_dot_product_attention(Q, K, V, attn_mask=attention_mask, is_causal=False) + A = A.transpose(1, 2).contiguous() + + attn_output = A.reshape(bsz, q_len, n_heads * head_dim) + attn_output = self.apply_o(self, attn_output) + # Optional deterministic residual dropout after attention projection + resid_attn_dropout = getattr(self, "_unsloth_resid_attn_dropout", None) + if resid_attn_dropout is not None and self.training: + attn_output = resid_attn_dropout(attn_output) + attn_weights = None + return attn_output, attn_weights, past_key_value + + +class FastPhiModel(FastLlamaModel): + + @staticmethod + def pre_patch(): + if PhiAttention is None: + return + # Patch attention forward for partial RoPE support and Unsloth compute path + PhiAttention .forward = PhiAttention_fast_forward + try: + PhiSdpaAttention .forward = PhiAttention_fast_forward + PhiFlashAttention2.forward = PhiAttention_fast_forward + except Exception: + pass + # Patch CausalLM for Unsloth fastpath when compatible + try: + PhiForCausalLM .forward = CausalLM_fast_forward(LlamaModel_fast_forward_inference) + PeftModelForCausalLM.forward = PeftModelForCausalLM_fast_forward + fix_prepare_inputs_for_generation(PhiForCausalLM) + except Exception: + pass + return + + @staticmethod + def post_patch(model, tokenizer): + # Ensure Phi-2 defaults for partial RoPE if missing in config + try: + if getattr(model.config, "model_type", None) == "phi": + if not hasattr(model.config, "partial_rotary_factor") and not hasattr(model.config, "rotary_dim"): + # Empirically common fraction for Phi-2 partial RoPE + model.config.partial_rotary_factor = 0.4 + # Attach deterministic dropout layers if dropout > 0 for residuals + p_attn = float(getattr(model.config, "attention_dropout", 0.0)) + p_mlp = float(getattr(model.config, "hidden_dropout", 0.0)) + if p_attn > 0.0 or p_mlp > 0.0: + import os + seed = int(os.environ.get("UNSLOTH_DROPOUT_SEED", 3407)) + for layer in model.model.layers: + if p_attn > 0.0: + # Attach to attention module so it can be used in patched attention forward + layer.self_attn._unsloth_resid_attn_dropout = DeterministicDropout(p_attn, seed) + if p_mlp > 0.0: + # Apply dropout to MLP outputs via a forward hook (post-MLP, pre-residual add) + layer._unsloth_resid_mlp_dropout = DeterministicDropout(p_mlp, seed) + def _mlp_hook(mod, inputs, output, _layer=layer): + if _layer.training and _layer._unsloth_resid_mlp_dropout is not None: + return _layer._unsloth_resid_mlp_dropout(output) + return output + # Keep handle to prevent GC + layer._unsloth_mlp_hook = layer.mlp.register_forward_hook(_mlp_hook) + except Exception: + pass + return model, tokenizer + + @staticmethod + def get_peft_model( + model, + r: int = 16, + target_modules = "all-linear", + lora_alpha: int = 16, + lora_dropout: float = 0.0, + bias: str = "none", + layers_to_transform = None, + layers_pattern = None, + use_gradient_checkpointing = True, + random_state: int = 3407, + max_seq_length: int = 2048, + use_rslora: bool = False, + modules_to_save = None, + init_lora_weights: bool = True, + loftq_config: dict = {}, + temporary_location: str = "_unsloth_temporary_saved_buffers", + **kwargs, + ): + return FastBaseModel.get_peft_model( + model = model, + r = r, + target_modules = target_modules, + lora_alpha = lora_alpha, + lora_dropout = lora_dropout, + bias = bias, + layers_to_transform = layers_to_transform, + layers_pattern = layers_pattern, + use_gradient_checkpointing = use_gradient_checkpointing, + random_state = random_state, + max_seq_length = max_seq_length, + use_rslora = use_rslora, + modules_to_save = modules_to_save, + init_lora_weights = init_lora_weights, + loftq_config = loftq_config, + temporary_location = temporary_location, + **kwargs, + ) + + @staticmethod + def from_pretrained( + model_name: str = "microsoft/Phi-2", + max_seq_length: Optional[int] = None, + dtype = None, + load_in_4bit: bool = True, + token: Optional[str] = None, + device_map: str = "sequential", + rope_scaling = None, + fix_tokenizer: bool = True, + model_patcher = None, + tokenizer_name: Optional[str] = None, + trust_remote_code: bool = False, + **kwargs, + ): + return FastLlamaModel.from_pretrained( + model_name = model_name, + max_seq_length = max_seq_length, + dtype = dtype, + load_in_4bit = load_in_4bit, + token = token, + device_map = device_map, + rope_scaling = rope_scaling, + fix_tokenizer = fix_tokenizer, + model_patcher = FastPhiModel, + tokenizer_name = tokenizer_name, + trust_remote_code = trust_remote_code, + **kwargs, + ) + pass +pass + + From cd9cae3eda48176f861199b4608335a6b308ff4a Mon Sep 17 00:00:00 2001 From: yash solanki Date: Wed, 13 Aug 2025 12:01:53 +0530 Subject: [PATCH 2/4] chore(review): remove unused GeLU exports, dedupe fast_layernorm_inference import; minor cleanup --- unsloth/kernels/__init__.py | 2 +- unsloth/kernels/gelu.py | 26 ++++---------------------- unsloth/models/cohere.py | 1 + 3 files changed, 6 insertions(+), 23 deletions(-) diff --git a/unsloth/kernels/__init__.py b/unsloth/kernels/__init__.py index 5b906b6cfa..de8b0666cf 100644 --- a/unsloth/kernels/__init__.py +++ b/unsloth/kernels/__init__.py @@ -29,7 +29,7 @@ from .layernorm import ( ) from .rope_embedding import fast_rope_embedding, inplace_rope_embedding from .dropout import DeterministicDropout, seeded_dropout -from .gelu import fast_gelu, FastGELU +# GeLU acceleration reserved; currently unused from .swiglu import swiglu_fg_kernel, swiglu_DWf_DW_dfg_kernel from .geglu import ( geglu_exact_forward_kernel, diff --git a/unsloth/kernels/gelu.py b/unsloth/kernels/gelu.py index c3b0499c3b..1cd209e5fa 100644 --- a/unsloth/kernels/gelu.py +++ b/unsloth/kernels/gelu.py @@ -12,27 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import torch -import torch.nn.functional as F - - -@torch.compiler.disable -def fast_gelu(x: torch.Tensor, approximate: str | None = None) -> torch.Tensor: - """Fast GeLU wrapper. Uses torch.nn.functional.gelu with optional approximation. - - approximate: None | "tanh" - """ - if approximate is None: - return F.gelu(x) - return F.gelu(x, approximate=approximate) - - -class FastGELU(torch.nn.Module): - def __init__(self, approximate: str | None = None): - super().__init__() - self.approximate = approximate - - def forward(self, x: torch.Tensor) -> torch.Tensor: - return fast_gelu(x, self.approximate) +""" +Reserved module for optional GeLU acceleration. Currently unused. +Left intentionally minimal to address reviewer feedback. +""" diff --git a/unsloth/models/cohere.py b/unsloth/models/cohere.py index 4251f3acd9..eb6a1accfc 100644 --- a/unsloth/models/cohere.py +++ b/unsloth/models/cohere.py @@ -13,6 +13,7 @@ # limitations under the License. from .llama import * +from ..kernels import fast_layernorm_inference from ._utils import __version__ from unsloth_zoo.hf_utils import dtype_from_config from unsloth_zoo.utils import _get_dtype, Version From ac60db34038f764781f87e6ba7876b914057a715 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 12 Mar 2026 08:18:05 +0000 Subject: [PATCH 3/4] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../test_unsloth_qlora_train_and_merge.py | 12 +- unsloth/kernels/__init__.py | 1 + unsloth/kernels/dropout.py | 35 ++-- unsloth/kernels/gelu.py | 2 - unsloth/kernels/layernorm.py | 12 +- unsloth/kernels/rope_embedding.py | 9 +- unsloth/models/phi.py | 156 +++++++++++------- 7 files changed, 141 insertions(+), 86 deletions(-) diff --git a/tests/qlora/test_unsloth_qlora_train_and_merge.py b/tests/qlora/test_unsloth_qlora_train_and_merge.py index 953aefcca7..502f79f4de 100644 --- a/tests/qlora/test_unsloth_qlora_train_and_merge.py +++ b/tests/qlora/test_unsloth_qlora_train_and_merge.py @@ -222,15 +222,17 @@ def test_unsloth_phi2_load_and_forward_smoke(): try: model, tokenizer = FastLanguageModel.from_pretrained( model_name, - max_seq_length=64, - load_in_4bit=True, - use_exact_model_name=True, + max_seq_length = 64, + load_in_4bit = True, + use_exact_model_name = True, ) except Exception as e: pytest.skip(f"Skipping Phi-2 smoke test due to: {e}") model.eval() with torch.no_grad(): - input_ids = tokenizer("Hello", return_tensors="pt").input_ids.to(next(model.parameters()).device) - out = model(input_ids=input_ids) + input_ids = tokenizer("Hello", return_tensors = "pt").input_ids.to( + next(model.parameters()).device + ) + out = model(input_ids = input_ids) assert hasattr(out, "logits") diff --git a/unsloth/kernels/__init__.py b/unsloth/kernels/__init__.py index de8b0666cf..31d74c2951 100644 --- a/unsloth/kernels/__init__.py +++ b/unsloth/kernels/__init__.py @@ -29,6 +29,7 @@ from .layernorm import ( ) from .rope_embedding import fast_rope_embedding, inplace_rope_embedding from .dropout import DeterministicDropout, seeded_dropout + # GeLU acceleration reserved; currently unused from .swiglu import swiglu_fg_kernel, swiglu_DWf_DW_dfg_kernel from .geglu import ( diff --git a/unsloth/kernels/dropout.py b/unsloth/kernels/dropout.py index 66563ce1e8..8dbcc1b9e1 100644 --- a/unsloth/kernels/dropout.py +++ b/unsloth/kernels/dropout.py @@ -20,28 +20,37 @@ _UINT32_MAX_INV = 1.0 / 4294967295.0 def _mix32(x: torch.Tensor) -> torch.Tensor: x = (x + 0x9E3779B9) & 0xFFFFFFFF - x ^= (x >> 16) + x ^= x >> 16 x = (x * 0x85EBCA6B) & 0xFFFFFFFF - x ^= (x >> 13) + x ^= x >> 13 x = (x * 0xC2B2AE35) & 0xFFFFFFFF - x ^= (x >> 16) + x ^= x >> 16 return x @torch.compiler.disable -def seeded_dropout(x: torch.Tensor, p: float, seed: int, scale: bool = True) -> torch.Tensor: - if p <= 0.0 or not (x.requires_grad or x.training if hasattr(x, 'training') else True): +def seeded_dropout( + x: torch.Tensor, p: float, seed: int, scale: bool = True +) -> torch.Tensor: + if p <= 0.0 or not ( + x.requires_grad or x.training if hasattr(x, "training") else True + ): return x device = x.device dtype = x.dtype bsz, seqlen, hidden = x.shape[0], x.shape[1], x.shape[-1] # Indices grids (broadcasted), keep memory modest by composing increments - b_idx = torch.arange(bsz, device=device, dtype=torch.int64).view(bsz, 1, 1) - t_idx = torch.arange(seqlen, device=device, dtype=torch.int64).view(1, seqlen, 1) - c_idx = torch.arange(hidden, device=device, dtype=torch.int64).view(1, 1, hidden) + b_idx = torch.arange(bsz, device = device, dtype = torch.int64).view(bsz, 1, 1) + t_idx = torch.arange(seqlen, device = device, dtype = torch.int64).view(1, seqlen, 1) + c_idx = torch.arange(hidden, device = device, dtype = torch.int64).view(1, 1, hidden) # Large coprime-like multipliers for mixing - mixed = (b_idx * 0x1F123BB5 + t_idx * 0x5DEECE66D + c_idx * 0xB5297A4D + (seed & 0xFFFFFFFF)) & 0xFFFFFFFF + mixed = ( + b_idx * 0x1F123BB5 + + t_idx * 0x5DEECE66D + + c_idx * 0xB5297A4D + + (seed & 0xFFFFFFFF) + ) & 0xFFFFFFFF rnd = _mix32(mixed).to(torch.float32) * _UINT32_MAX_INV mask = (rnd >= p).to(dtype) if scale and p < 1.0: @@ -56,7 +65,9 @@ class DeterministicDropout(torch.nn.Module): # Allow override via env variable env_seed = os.environ.get("UNSLOTH_DROPOUT_SEED", None) self.seed = int(env_seed) if env_seed is not None else int(seed) - self.register_buffer('_counter', torch.zeros((), dtype=torch.int64), persistent=False) + self.register_buffer( + "_counter", torch.zeros((), dtype = torch.int64), persistent = False + ) def forward(self, x: torch.Tensor) -> torch.Tensor: if (not self.training) or self.p <= 0.0: @@ -65,6 +76,4 @@ class DeterministicDropout(torch.nn.Module): local = int(self._counter.item()) self._counter.add_(1) derived_seed = (self.seed + 0x9E3779B9 * local) & 0xFFFFFFFF - return seeded_dropout(x, self.p, derived_seed, scale=True) - - + return seeded_dropout(x, self.p, derived_seed, scale = True) diff --git a/unsloth/kernels/gelu.py b/unsloth/kernels/gelu.py index 1cd209e5fa..4fa6e673f4 100644 --- a/unsloth/kernels/gelu.py +++ b/unsloth/kernels/gelu.py @@ -16,5 +16,3 @@ Reserved module for optional GeLU acceleration. Currently unused. Left intentionally minimal to address reviewer feedback. """ - - diff --git a/unsloth/kernels/layernorm.py b/unsloth/kernels/layernorm.py index a09a541aff..0992a0416a 100644 --- a/unsloth/kernels/layernorm.py +++ b/unsloth/kernels/layernorm.py @@ -181,13 +181,16 @@ def fast_layernorm(layernorm, X): out = Fast_Layernorm.apply(X, W, bias, eps) return out + # Public helper mirroring RMSNorm API for standard LayerNorm cases @torch.compiler.disable -def fast_layernorm_inference(layernorm, X: torch.Tensor, out_weight: torch.Tensor | None = None): - XX = X.to(torch.float32, copy=True) - mean = XX.mean(-1, keepdim=True) +def fast_layernorm_inference( + layernorm, X: torch.Tensor, out_weight: torch.Tensor | None = None +): + XX = X.to(torch.float32, copy = True) + mean = XX.mean(-1, keepdim = True) XX -= mean - var = (XX * XX).mean(-1, keepdim=True) + var = (XX * XX).mean(-1, keepdim = True) var += layernorm.eps if hasattr(layernorm, "eps") else layernorm.variance_epsilon XX *= var.rsqrt_() if out_weight is None: @@ -196,7 +199,6 @@ def fast_layernorm_inference(layernorm, X: torch.Tensor, out_weight: torch.Tenso return (XX * out_weight).to(X.dtype) - def test_layernorm( dim = 1024, eps = 1e-5, diff --git a/unsloth/kernels/rope_embedding.py b/unsloth/kernels/rope_embedding.py index f1b6c363ae..5b03111cb6 100644 --- a/unsloth/kernels/rope_embedding.py +++ b/unsloth/kernels/rope_embedding.py @@ -463,7 +463,8 @@ def inplace_rope_embedding(Q, K, cos, sin, position_ids): K = Slow_RoPE_Embedding.apply(K, cos, sin, position_ids) torch_device_stream(Q.device).synchronize() return Q, K -pass + + @torch.compiler.disable @@ -483,7 +484,8 @@ def fast_partial_rope_embedding(Q, K, cos, sin, rotary_dim: int): Q[..., :rotary_dim] = Q_rot2 K[..., :rotary_dim] = K_rot2 return Q, K -pass + + def inplace_partial_rope_embedding(Q, K, cos, sin, position_ids, rotary_dim: int): @@ -502,4 +504,5 @@ def inplace_partial_rope_embedding(Q, K, cos, sin, position_ids, rotary_dim: int Q[..., :rotary_dim] = Q_rot2 K[..., :rotary_dim] = K_rot2 return Q, K -pass + + diff --git a/unsloth/models/phi.py b/unsloth/models/phi.py index 0ff46cb2b1..cb832d53d7 100644 --- a/unsloth/models/phi.py +++ b/unsloth/models/phi.py @@ -23,6 +23,7 @@ from typing import Optional, Tuple try: from transformers import __version__ as transformers_version + transformers_version = Version(transformers_version) from transformers.models.phi.modeling_phi import ( PhiAttention, @@ -30,8 +31,12 @@ try: PhiModel, PhiForCausalLM, ) + try: - from transformers.models.phi.modeling_phi import PhiSdpaAttention, PhiFlashAttention2 + from transformers.models.phi.modeling_phi import ( + PhiSdpaAttention, + PhiFlashAttention2, + ) except Exception: PhiSdpaAttention = PhiAttention PhiFlashAttention2 = PhiAttention @@ -72,9 +77,9 @@ def PhiAttention_fast_forward( use_cache: bool = False, padding_mask: Optional[torch.LongTensor] = None, position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, - *args, **kwargs, + *args, + **kwargs, ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: - # Clear inference caches if any (mirrors other model fastpaths) if hasattr(self, "paged_attention"): del self.paged_attention_K @@ -84,16 +89,19 @@ def PhiAttention_fast_forward( del self.temp_KV del self.RH_Q del self.attention - pass bsz, q_len, _ = hidden_states.size() n_heads: int = self.config.num_attention_heads n_kv_heads: int = getattr(self.config, "num_key_value_heads", n_heads) n_groups_attr = getattr(self, "num_key_value_groups", None) - n_groups: int = n_groups_attr if isinstance(n_groups_attr, int) and n_groups_attr > 0 else max(1, n_heads // max(1, n_kv_heads)) + n_groups: int = ( + n_groups_attr + if isinstance(n_groups_attr, int) and n_groups_attr > 0 + else max(1, n_heads // max(1, n_kv_heads)) + ) head_dim: int = self.head_dim - assert (n_kv_heads * n_groups == n_heads) + assert n_kv_heads * n_groups == n_heads # Q, K, V projections Q, K, V = self.apply_qkv(self, hidden_states) @@ -133,12 +141,12 @@ def PhiAttention_fast_forward( rope_module = None if hasattr(self, "rotary_emb"): rope_module = self.rotary_emb - rope_module.extend_rope_embedding(V, seq_len=kv_seq_len) + rope_module.extend_rope_embedding(V, seq_len = kv_seq_len) if position_ids is None: cos = rope_module.cos_cached sin = rope_module.sin_cached else: - cos, sin = rope_module(V, seq_len=kv_seq_len) + cos, sin = rope_module(V, seq_len = kv_seq_len) else: rope_module = getattr(self, "_unsloth_phi_rope", None) if rope_module is None: @@ -149,16 +157,21 @@ def PhiAttention_fast_forward( except Exception: base = 10000 max_pos = 2048 - rope_module = LlamaRotaryEmbedding(dim=head_dim, max_position_embeddings=max_pos, base=base, device=V.device) + rope_module = LlamaRotaryEmbedding( + dim = head_dim, + max_position_embeddings = max_pos, + base = base, + device = V.device, + ) # Keep for reuse self._unsloth_phi_rope = rope_module # Ensure buffers sized appropriately - rope_module.extend_rope_embedding(V, seq_len=kv_seq_len) + rope_module.extend_rope_embedding(V, seq_len = kv_seq_len) if position_ids is None: cos = rope_module.cos_cached sin = rope_module.sin_cached else: - cos, sin = rope_module(V, seq_len=kv_seq_len) + cos, sin = rope_module(V, seq_len = kv_seq_len) # Apply (partial) RoPE if rotary_dim < head_dim: @@ -172,12 +185,12 @@ def PhiAttention_fast_forward( # KV cache update if past_key_value is not None: - K = torch.cat([past_key_value[0], K], dim=2) - V = torch.cat([past_key_value[1], V], dim=2) + K = torch.cat([past_key_value[0], K], dim = 2) + V = torch.cat([past_key_value[1], V], dim = 2) past_key_value = (K, V) if use_cache else None # Attention computation (dispatch as in other models) - if (not HAS_FLASH_ATTENTION and HAS_XFORMERS and attention_mask is None): + if not HAS_FLASH_ATTENTION and HAS_XFORMERS and attention_mask is None: # Xformers memory efficient attention with (bsz, seqlen, heads, dim) Q = Q.transpose(1, 2) K = K.transpose(1, 2) @@ -194,27 +207,40 @@ def PhiAttention_fast_forward( V = V.reshape(bsz, kv_seq_len, n_heads, head_dim) else: Q = Q.view(bsz, q_len, n_kv_heads, n_groups, head_dim) - A = xformers_attention(Q, K, V, attn_bias=causal_mask) + A = xformers_attention(Q, K, V, attn_bias = causal_mask) A = A.view(bsz, q_len, n_heads, head_dim) elif HAS_FLASH_ATTENTION and attention_mask is None: Q = Q.transpose(1, 2) K = K.transpose(1, 2) V = V.transpose(1, 2) - A = flash_attn_func(Q, K, V, causal=True) + A = flash_attn_func(Q, K, V, causal = True) else: # SDPA fallback, support GQA if available if SDPA_HAS_GQA: - A = scaled_dot_product_attention(Q, K, V, attn_mask=attention_mask, is_causal=False, enable_gqa=n_groups != 1) + A = scaled_dot_product_attention( + Q, + K, + V, + attn_mask = attention_mask, + is_causal = False, + enable_gqa = n_groups != 1, + ) A = A.transpose(1, 2) else: if n_groups != 1: - K = K[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, kv_seq_len, head_dim) - V = V[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, kv_seq_len, head_dim) + K = K[:, :, None, :, :].expand( + bsz, n_kv_heads, n_groups, kv_seq_len, head_dim + ) + V = V[:, :, None, :, :].expand( + bsz, n_kv_heads, n_groups, kv_seq_len, head_dim + ) K = K.reshape(bsz, n_heads, kv_seq_len, head_dim) V = V.reshape(bsz, n_heads, kv_seq_len, head_dim) Q, K, V = Q.contiguous(), K.contiguous(), V.contiguous() - A = scaled_dot_product_attention(Q, K, V, attn_mask=attention_mask, is_causal=False) + A = scaled_dot_product_attention( + Q, K, V, attn_mask = attention_mask, is_causal = False + ) A = A.transpose(1, 2).contiguous() attn_output = A.reshape(bsz, q_len, n_heads * head_dim) @@ -228,21 +254,22 @@ def PhiAttention_fast_forward( class FastPhiModel(FastLlamaModel): - @staticmethod def pre_patch(): if PhiAttention is None: return # Patch attention forward for partial RoPE support and Unsloth compute path - PhiAttention .forward = PhiAttention_fast_forward + PhiAttention.forward = PhiAttention_fast_forward try: - PhiSdpaAttention .forward = PhiAttention_fast_forward + PhiSdpaAttention.forward = PhiAttention_fast_forward PhiFlashAttention2.forward = PhiAttention_fast_forward except Exception: pass # Patch CausalLM for Unsloth fastpath when compatible try: - PhiForCausalLM .forward = CausalLM_fast_forward(LlamaModel_fast_forward_inference) + PhiForCausalLM.forward = CausalLM_fast_forward( + LlamaModel_fast_forward_inference + ) PeftModelForCausalLM.forward = PeftModelForCausalLM_fast_forward fix_prepare_inputs_for_generation(PhiForCausalLM) except Exception: @@ -254,28 +281,42 @@ class FastPhiModel(FastLlamaModel): # Ensure Phi-2 defaults for partial RoPE if missing in config try: if getattr(model.config, "model_type", None) == "phi": - if not hasattr(model.config, "partial_rotary_factor") and not hasattr(model.config, "rotary_dim"): + if not hasattr(model.config, "partial_rotary_factor") and not hasattr( + model.config, "rotary_dim" + ): # Empirically common fraction for Phi-2 partial RoPE model.config.partial_rotary_factor = 0.4 # Attach deterministic dropout layers if dropout > 0 for residuals p_attn = float(getattr(model.config, "attention_dropout", 0.0)) - p_mlp = float(getattr(model.config, "hidden_dropout", 0.0)) + p_mlp = float(getattr(model.config, "hidden_dropout", 0.0)) if p_attn > 0.0 or p_mlp > 0.0: import os + seed = int(os.environ.get("UNSLOTH_DROPOUT_SEED", 3407)) for layer in model.model.layers: if p_attn > 0.0: # Attach to attention module so it can be used in patched attention forward - layer.self_attn._unsloth_resid_attn_dropout = DeterministicDropout(p_attn, seed) + layer.self_attn._unsloth_resid_attn_dropout = ( + DeterministicDropout(p_attn, seed) + ) if p_mlp > 0.0: # Apply dropout to MLP outputs via a forward hook (post-MLP, pre-residual add) - layer._unsloth_resid_mlp_dropout = DeterministicDropout(p_mlp, seed) - def _mlp_hook(mod, inputs, output, _layer=layer): - if _layer.training and _layer._unsloth_resid_mlp_dropout is not None: + layer._unsloth_resid_mlp_dropout = DeterministicDropout( + p_mlp, seed + ) + + def _mlp_hook(mod, inputs, output, _layer = layer): + if ( + _layer.training + and _layer._unsloth_resid_mlp_dropout is not None + ): return _layer._unsloth_resid_mlp_dropout(output) return output + # Keep handle to prevent GC - layer._unsloth_mlp_hook = layer.mlp.register_forward_hook(_mlp_hook) + layer._unsloth_mlp_hook = layer.mlp.register_forward_hook( + _mlp_hook + ) except Exception: pass return model, tokenizer @@ -301,22 +342,22 @@ class FastPhiModel(FastLlamaModel): **kwargs, ): return FastBaseModel.get_peft_model( - model = model, - r = r, - target_modules = target_modules, - lora_alpha = lora_alpha, - lora_dropout = lora_dropout, - bias = bias, - layers_to_transform = layers_to_transform, - layers_pattern = layers_pattern, + model = model, + r = r, + target_modules = target_modules, + lora_alpha = lora_alpha, + lora_dropout = lora_dropout, + bias = bias, + layers_to_transform = layers_to_transform, + layers_pattern = layers_pattern, use_gradient_checkpointing = use_gradient_checkpointing, - random_state = random_state, - max_seq_length = max_seq_length, - use_rslora = use_rslora, - modules_to_save = modules_to_save, - init_lora_weights = init_lora_weights, - loftq_config = loftq_config, - temporary_location = temporary_location, + random_state = random_state, + max_seq_length = max_seq_length, + use_rslora = use_rslora, + modules_to_save = modules_to_save, + init_lora_weights = init_lora_weights, + loftq_config = loftq_config, + temporary_location = temporary_location, **kwargs, ) @@ -336,20 +377,19 @@ class FastPhiModel(FastLlamaModel): **kwargs, ): return FastLlamaModel.from_pretrained( - model_name = model_name, - max_seq_length = max_seq_length, - dtype = dtype, - load_in_4bit = load_in_4bit, - token = token, - device_map = device_map, - rope_scaling = rope_scaling, - fix_tokenizer = fix_tokenizer, - model_patcher = FastPhiModel, - tokenizer_name = tokenizer_name, + model_name = model_name, + max_seq_length = max_seq_length, + dtype = dtype, + load_in_4bit = load_in_4bit, + token = token, + device_map = device_map, + rope_scaling = rope_scaling, + fix_tokenizer = fix_tokenizer, + model_patcher = FastPhiModel, + tokenizer_name = tokenizer_name, trust_remote_code = trust_remote_code, **kwargs, ) - pass -pass + From c6a7ba5e28423d3cc62cdaace410b5bf323b7005 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 12 Mar 2026 23:11:16 +0000 Subject: [PATCH 4/4] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- unsloth/kernels/rope_embedding.py | 6 ------ unsloth/models/phi.py | 3 --- 2 files changed, 9 deletions(-) diff --git a/unsloth/kernels/rope_embedding.py b/unsloth/kernels/rope_embedding.py index 5b03111cb6..d311ab3791 100644 --- a/unsloth/kernels/rope_embedding.py +++ b/unsloth/kernels/rope_embedding.py @@ -465,8 +465,6 @@ def inplace_rope_embedding(Q, K, cos, sin, position_ids): return Q, K - - @torch.compiler.disable def fast_partial_rope_embedding(Q, K, cos, sin, rotary_dim: int): """Apply RoPE only to the first rotary_dim features of Q and K using the fast kernel. @@ -486,8 +484,6 @@ def fast_partial_rope_embedding(Q, K, cos, sin, rotary_dim: int): return Q, K - - def inplace_partial_rope_embedding(Q, K, cos, sin, position_ids, rotary_dim: int): """Apply RoPE only to the first rotary_dim features of Q and K using the slow kernel. @@ -504,5 +500,3 @@ def inplace_partial_rope_embedding(Q, K, cos, sin, position_ids, rotary_dim: int Q[..., :rotary_dim] = Q_rot2 K[..., :rotary_dim] = K_rot2 return Q, K - - diff --git a/unsloth/models/phi.py b/unsloth/models/phi.py index cb832d53d7..be0bc3b62f 100644 --- a/unsloth/models/phi.py +++ b/unsloth/models/phi.py @@ -390,6 +390,3 @@ class FastPhiModel(FastLlamaModel): trust_remote_code = trust_remote_code, **kwargs, ) - - -