diff --git a/tests/qlora/test_unsloth_qlora_train_and_merge.py b/tests/qlora/test_unsloth_qlora_train_and_merge.py index 9040ad793d..502f79f4de 100644 --- a/tests/qlora/test_unsloth_qlora_train_and_merge.py +++ b/tests/qlora/test_unsloth_qlora_train_and_merge.py @@ -209,3 +209,30 @@ if __name__ == "__main__": ) with header_footer_context("Responses after unsloth merge to 16bit"): check_responses(responses, answer = ANSWER, prompt = prompt) + + +# Minimal Phi-2 smoke test to ensure loader + forward path works. +# Skips automatically if model cannot be downloaded in CI. +def test_unsloth_phi2_load_and_forward_smoke(): + import pytest + import torch + from unsloth import FastLanguageModel + + model_name = "microsoft/Phi-2" + try: + model, tokenizer = FastLanguageModel.from_pretrained( + model_name, + max_seq_length = 64, + load_in_4bit = True, + use_exact_model_name = True, + ) + except Exception as e: + pytest.skip(f"Skipping Phi-2 smoke test due to: {e}") + + model.eval() + with torch.no_grad(): + input_ids = tokenizer("Hello", return_tensors = "pt").input_ids.to( + next(model.parameters()).device + ) + out = model(input_ids = input_ids) + assert hasattr(out, "logits") diff --git a/unsloth/kernels/__init__.py b/unsloth/kernels/__init__.py index 15913413d9..31d74c2951 100644 --- a/unsloth/kernels/__init__.py +++ b/unsloth/kernels/__init__.py @@ -25,8 +25,12 @@ from .rms_layernorm import ( from .layernorm import ( fast_layernorm, patch_layernorm, + fast_layernorm_inference, ) from .rope_embedding import fast_rope_embedding, inplace_rope_embedding +from .dropout import DeterministicDropout, seeded_dropout + +# GeLU acceleration reserved; currently unused from .swiglu import swiglu_fg_kernel, swiglu_DWf_DW_dfg_kernel from .geglu import ( geglu_exact_forward_kernel, diff --git a/unsloth/kernels/dropout.py b/unsloth/kernels/dropout.py new file mode 100644 index 0000000000..8dbcc1b9e1 --- /dev/null +++ b/unsloth/kernels/dropout.py @@ -0,0 +1,79 @@ +# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import torch + +_UINT32_MAX_INV = 1.0 / 4294967295.0 + + +def _mix32(x: torch.Tensor) -> torch.Tensor: + x = (x + 0x9E3779B9) & 0xFFFFFFFF + x ^= x >> 16 + x = (x * 0x85EBCA6B) & 0xFFFFFFFF + x ^= x >> 13 + x = (x * 0xC2B2AE35) & 0xFFFFFFFF + x ^= x >> 16 + return x + + +@torch.compiler.disable +def seeded_dropout( + x: torch.Tensor, p: float, seed: int, scale: bool = True +) -> torch.Tensor: + if p <= 0.0 or not ( + x.requires_grad or x.training if hasattr(x, "training") else True + ): + return x + device = x.device + dtype = x.dtype + bsz, seqlen, hidden = x.shape[0], x.shape[1], x.shape[-1] + # Indices grids (broadcasted), keep memory modest by composing increments + b_idx = torch.arange(bsz, device = device, dtype = torch.int64).view(bsz, 1, 1) + t_idx = torch.arange(seqlen, device = device, dtype = torch.int64).view(1, seqlen, 1) + c_idx = torch.arange(hidden, device = device, dtype = torch.int64).view(1, 1, hidden) + + # Large coprime-like multipliers for mixing + mixed = ( + b_idx * 0x1F123BB5 + + t_idx * 0x5DEECE66D + + c_idx * 0xB5297A4D + + (seed & 0xFFFFFFFF) + ) & 0xFFFFFFFF + rnd = _mix32(mixed).to(torch.float32) * _UINT32_MAX_INV + mask = (rnd >= p).to(dtype) + if scale and p < 1.0: + mask = mask / (1.0 - p) + return x * mask + + +class DeterministicDropout(torch.nn.Module): + def __init__(self, p: float, seed: int = 3407): + super().__init__() + self.p = float(p) + # Allow override via env variable + env_seed = os.environ.get("UNSLOTH_DROPOUT_SEED", None) + self.seed = int(env_seed) if env_seed is not None else int(seed) + self.register_buffer( + "_counter", torch.zeros((), dtype = torch.int64), persistent = False + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + if (not self.training) or self.p <= 0.0: + return x + # Derive a new seed per call to decorrelate successive uses + local = int(self._counter.item()) + self._counter.add_(1) + derived_seed = (self.seed + 0x9E3779B9 * local) & 0xFFFFFFFF + return seeded_dropout(x, self.p, derived_seed, scale = True) diff --git a/unsloth/kernels/gelu.py b/unsloth/kernels/gelu.py new file mode 100644 index 0000000000..4fa6e673f4 --- /dev/null +++ b/unsloth/kernels/gelu.py @@ -0,0 +1,18 @@ +# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Reserved module for optional GeLU acceleration. Currently unused. +Left intentionally minimal to address reviewer feedback. +""" diff --git a/unsloth/kernels/layernorm.py b/unsloth/kernels/layernorm.py index 9e64c3d341..0992a0416a 100644 --- a/unsloth/kernels/layernorm.py +++ b/unsloth/kernels/layernorm.py @@ -182,6 +182,23 @@ def fast_layernorm(layernorm, X): return out +# Public helper mirroring RMSNorm API for standard LayerNorm cases +@torch.compiler.disable +def fast_layernorm_inference( + layernorm, X: torch.Tensor, out_weight: torch.Tensor | None = None +): + XX = X.to(torch.float32, copy = True) + mean = XX.mean(-1, keepdim = True) + XX -= mean + var = (XX * XX).mean(-1, keepdim = True) + var += layernorm.eps if hasattr(layernorm, "eps") else layernorm.variance_epsilon + XX *= var.rsqrt_() + if out_weight is None: + return (XX * layernorm.weight).to(X.dtype) + out_weight[:] = layernorm.weight + return (XX * out_weight).to(X.dtype) + + def test_layernorm( dim = 1024, eps = 1e-5, diff --git a/unsloth/kernels/rope_embedding.py b/unsloth/kernels/rope_embedding.py index fcc9cb923b..d311ab3791 100644 --- a/unsloth/kernels/rope_embedding.py +++ b/unsloth/kernels/rope_embedding.py @@ -463,3 +463,40 @@ def inplace_rope_embedding(Q, K, cos, sin, position_ids): K = Slow_RoPE_Embedding.apply(K, cos, sin, position_ids) torch_device_stream(Q.device).synchronize() return Q, K + + +@torch.compiler.disable +def fast_partial_rope_embedding(Q, K, cos, sin, rotary_dim: int): + """Apply RoPE only to the first rotary_dim features of Q and K using the fast kernel. + + Shapes: + Q, K: [bsz, n_heads, seqlen, head_dim] + cos, sin: broadcastable to [seqlen, rotary_dim] + """ + if rotary_dim <= 0: + return Q, K + Q_rot = Q[..., :rotary_dim] + K_rot = K[..., :rotary_dim] + Q_rot2 = Fast_RoPE_Embedding.apply(Q_rot.transpose(1, 2), cos, sin).transpose(1, 2) + K_rot2 = Fast_RoPE_Embedding.apply(K_rot.transpose(1, 2), cos, sin).transpose(1, 2) + Q[..., :rotary_dim] = Q_rot2 + K[..., :rotary_dim] = K_rot2 + return Q, K + + +def inplace_partial_rope_embedding(Q, K, cos, sin, position_ids, rotary_dim: int): + """Apply RoPE only to the first rotary_dim features of Q and K using the slow kernel. + + Shapes: + Q, K: [bsz, n_heads, seqlen, head_dim] + cos, sin: broadcastable to [seqlen, rotary_dim] + """ + if rotary_dim <= 0: + return Q, K + Q_rot = Q[..., :rotary_dim] + K_rot = K[..., :rotary_dim] + Q_rot2 = Slow_RoPE_Embedding.apply(Q_rot, cos, sin, position_ids) + K_rot2 = Slow_RoPE_Embedding.apply(K_rot, cos, sin, position_ids) + Q[..., :rotary_dim] = Q_rot2 + K[..., :rotary_dim] = K_rot2 + return Q, K diff --git a/unsloth/models/__init__.py b/unsloth/models/__init__.py index 138f309032..7b31d9fbc6 100644 --- a/unsloth/models/__init__.py +++ b/unsloth/models/__init__.py @@ -20,6 +20,7 @@ from .qwen3 import FastQwen3Model from .qwen3_moe import FastQwen3MoeModel from .granite import FastGraniteModel from .sentence_transformer import FastSentenceTransformer +from .phi import FastPhiModel try: from .falcon_h1 import FastFalconH1Model diff --git a/unsloth/models/cohere.py b/unsloth/models/cohere.py index 4251f3acd9..eb6a1accfc 100644 --- a/unsloth/models/cohere.py +++ b/unsloth/models/cohere.py @@ -13,6 +13,7 @@ # limitations under the License. from .llama import * +from ..kernels import fast_layernorm_inference from ._utils import __version__ from unsloth_zoo.hf_utils import dtype_from_config from unsloth_zoo.utils import _get_dtype, Version diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index bd15ed5281..e58259846d 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -29,6 +29,7 @@ from .qwen2 import FastQwen2Model from .qwen3 import FastQwen3Model from .qwen3_moe import FastQwen3MoeModel from .cohere import FastCohereModel +from .phi import FastPhiModel from transformers import AutoConfig from transformers import __version__ as transformers_version from peft import PeftConfig, PeftModel @@ -635,6 +636,8 @@ class FastLanguageModel(FastLlamaModel): # f'Try `pip install --upgrade "transformers>=4.50.3"`\n'\ # f"to obtain the latest transformers build, then restart this session."\ # ) + elif model_type == "phi": + dispatch_model = FastPhiModel # Temporary disable optimized Cohere until errors match # elif model_type == "cohere": # dispatch_model = FastCohereModel @@ -742,6 +745,12 @@ class FastLanguageModel(FastLlamaModel): ] ) + # Allow model-specific post patches (e.g., Phi-2 defaults) + try: + model, tokenizer = dispatch_model.post_patch(model, tokenizer) + except Exception: + pass + if load_in_4bit: # Fix up bitsandbytes config, but respect user-provided quantization_config if quantization_config is None: diff --git a/unsloth/models/mapper.py b/unsloth/models/mapper.py index f0f430eb7e..e2a8944035 100644 --- a/unsloth/models/mapper.py +++ b/unsloth/models/mapper.py @@ -609,6 +609,10 @@ __INT_TO_FLOAT_MAPPER = \ "microsoft/phi-4", "unsloth/phi-4-bnb-4bit", ), + "unsloth/Phi-2-bnb-4bit" : ( + "unsloth/Phi-2", + "microsoft/Phi-2", + ), "unsloth/DeepSeek-R1-Distill-Qwen-32B-bnb-4bit" : ( "unsloth/DeepSeek-R1-Distill-Qwen-32B", "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B", diff --git a/unsloth/models/phi.py b/unsloth/models/phi.py new file mode 100644 index 0000000000..be0bc3b62f --- /dev/null +++ b/unsloth/models/phi.py @@ -0,0 +1,392 @@ +# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .llama import * +from ._utils import __version__ +from unsloth_zoo.utils import Version, _get_dtype +from .vision import FastBaseModel +from ..kernels import DeterministicDropout + +import torch +from typing import Optional, Tuple + +try: + from transformers import __version__ as transformers_version + + transformers_version = Version(transformers_version) + from transformers.models.phi.modeling_phi import ( + PhiAttention, + PhiDecoderLayer, + PhiModel, + PhiForCausalLM, + ) + + try: + from transformers.models.phi.modeling_phi import ( + PhiSdpaAttention, + PhiFlashAttention2, + ) + except Exception: + PhiSdpaAttention = PhiAttention + PhiFlashAttention2 = PhiAttention +except Exception as error: + # We only import when actually used; loader will guard by AutoConfig + PhiAttention = None + PhiDecoderLayer = None + PhiModel = None + PhiForCausalLM = None + PhiSdpaAttention = None + PhiFlashAttention2 = None + + +def _phi_get_rotary_dims(attn_module) -> int: + head_dim: int = attn_module.head_dim + # Prefer explicit rotary_dim if provided by config + rotary_dim = getattr(attn_module.config, "rotary_dim", None) + if isinstance(rotary_dim, int) and 0 < rotary_dim <= head_dim: + # Ensure even for half-rotate math + return (rotary_dim // 2) * 2 + # Else use partial_rotary_factor if present + fraction = getattr(attn_module.config, "partial_rotary_factor", None) + if isinstance(fraction, (float, int)) and 0 < fraction <= 1: + rotary_dim = int(head_dim * float(fraction)) + return (rotary_dim // 2) * 2 + # Default: full rotation + return (head_dim // 2) * 2 if head_dim % 2 != 0 else head_dim + + +def PhiAttention_fast_forward( + self, + hidden_states: torch.Tensor, + causal_mask: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Tuple[torch.Tensor]] = None, + output_attentions: bool = False, + use_cache: bool = False, + padding_mask: Optional[torch.LongTensor] = None, + position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + *args, + **kwargs, +) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + # Clear inference caches if any (mirrors other model fastpaths) + if hasattr(self, "paged_attention"): + del self.paged_attention_K + del self.paged_attention_V + del self.paged_attention + del self.temp_QA + del self.temp_KV + del self.RH_Q + del self.attention + + bsz, q_len, _ = hidden_states.size() + + n_heads: int = self.config.num_attention_heads + n_kv_heads: int = getattr(self.config, "num_key_value_heads", n_heads) + n_groups_attr = getattr(self, "num_key_value_groups", None) + n_groups: int = ( + n_groups_attr + if isinstance(n_groups_attr, int) and n_groups_attr > 0 + else max(1, n_heads // max(1, n_kv_heads)) + ) + head_dim: int = self.head_dim + assert n_kv_heads * n_groups == n_heads + + # Q, K, V projections + Q, K, V = self.apply_qkv(self, hidden_states) + Q = Q.view(bsz, q_len, n_heads, head_dim).transpose(1, 2) + K = K.view(bsz, q_len, n_kv_heads, head_dim).transpose(1, 2) + V = V.view(bsz, q_len, n_kv_heads, head_dim).transpose(1, 2) + + # Sequence lengths with KV cache + kv_seq_len = K.shape[-2] + if past_key_value is not None: + kv_seq_len += past_key_value[0].shape[-2] + + # Apply (partial) RoPE on first rotary_dim dims of Q, K + rotary_dim: int = _phi_get_rotary_dims(self) + if position_embeddings is not None: + cos, sin = position_embeddings + if rotary_dim < head_dim: + cos = cos[..., :rotary_dim] + sin = sin[..., :rotary_dim] + # Fast path when position_ids provided handled below via inplace op + if position_ids is None: + Q_rot = Q[..., :rotary_dim] + K_rot = K[..., :rotary_dim] + Q_rot, K_rot = inplace_rope_embedding(Q_rot, K_rot, cos, sin, position_ids) + Q[..., :rotary_dim] = Q_rot + K[..., :rotary_dim] = K_rot + else: + Q_rot = Q[..., :rotary_dim] + K_rot = K[..., :rotary_dim] + cos = cos[position_ids].unsqueeze(1) # [bs, 1, seqlen, dim] + sin = sin[position_ids].unsqueeze(1) + Q_rot, K_rot = inplace_rope_embedding(Q_rot, K_rot, cos, sin, position_ids) + Q[..., :rotary_dim] = Q_rot + K[..., :rotary_dim] = K_rot + else: + # Compute cos/sin from available rotary embedding; if none, create a local one + rope_module = None + if hasattr(self, "rotary_emb"): + rope_module = self.rotary_emb + rope_module.extend_rope_embedding(V, seq_len = kv_seq_len) + if position_ids is None: + cos = rope_module.cos_cached + sin = rope_module.sin_cached + else: + cos, sin = rope_module(V, seq_len = kv_seq_len) + else: + rope_module = getattr(self, "_unsloth_phi_rope", None) + if rope_module is None: + # Build Llama-style rotary embedding configured for Phi + try: + base = getattr(self.config, "rope_theta", 10000) + max_pos = getattr(self.config, "max_position_embeddings", 2048) + except Exception: + base = 10000 + max_pos = 2048 + rope_module = LlamaRotaryEmbedding( + dim = head_dim, + max_position_embeddings = max_pos, + base = base, + device = V.device, + ) + # Keep for reuse + self._unsloth_phi_rope = rope_module + # Ensure buffers sized appropriately + rope_module.extend_rope_embedding(V, seq_len = kv_seq_len) + if position_ids is None: + cos = rope_module.cos_cached + sin = rope_module.sin_cached + else: + cos, sin = rope_module(V, seq_len = kv_seq_len) + + # Apply (partial) RoPE + if rotary_dim < head_dim: + cos = cos[..., :rotary_dim] + sin = sin[..., :rotary_dim] + Q_rot = Q[..., :rotary_dim] + K_rot = K[..., :rotary_dim] + Q_rot, K_rot = inplace_rope_embedding(Q_rot, K_rot, cos, sin, position_ids) + Q[..., :rotary_dim] = Q_rot + K[..., :rotary_dim] = K_rot + + # KV cache update + if past_key_value is not None: + K = torch.cat([past_key_value[0], K], dim = 2) + V = torch.cat([past_key_value[1], V], dim = 2) + past_key_value = (K, V) if use_cache else None + + # Attention computation (dispatch as in other models) + if not HAS_FLASH_ATTENTION and HAS_XFORMERS and attention_mask is None: + # Xformers memory efficient attention with (bsz, seqlen, heads, dim) + Q = Q.transpose(1, 2) + K = K.transpose(1, 2) + V = V.transpose(1, 2) + + # Grouped Query Attention (expand KV across groups) + if n_groups != 1: + K = K.view(bsz, kv_seq_len, n_kv_heads, 1, head_dim) + V = V.view(bsz, kv_seq_len, n_kv_heads, 1, head_dim) + K = K.expand(bsz, kv_seq_len, n_kv_heads, n_groups, head_dim) + V = V.expand(bsz, kv_seq_len, n_kv_heads, n_groups, head_dim) + if hidden_states.requires_grad: + K = K.reshape(bsz, kv_seq_len, n_heads, head_dim) + V = V.reshape(bsz, kv_seq_len, n_heads, head_dim) + else: + Q = Q.view(bsz, q_len, n_kv_heads, n_groups, head_dim) + A = xformers_attention(Q, K, V, attn_bias = causal_mask) + A = A.view(bsz, q_len, n_heads, head_dim) + + elif HAS_FLASH_ATTENTION and attention_mask is None: + Q = Q.transpose(1, 2) + K = K.transpose(1, 2) + V = V.transpose(1, 2) + A = flash_attn_func(Q, K, V, causal = True) + else: + # SDPA fallback, support GQA if available + if SDPA_HAS_GQA: + A = scaled_dot_product_attention( + Q, + K, + V, + attn_mask = attention_mask, + is_causal = False, + enable_gqa = n_groups != 1, + ) + A = A.transpose(1, 2) + else: + if n_groups != 1: + K = K[:, :, None, :, :].expand( + bsz, n_kv_heads, n_groups, kv_seq_len, head_dim + ) + V = V[:, :, None, :, :].expand( + bsz, n_kv_heads, n_groups, kv_seq_len, head_dim + ) + K = K.reshape(bsz, n_heads, kv_seq_len, head_dim) + V = V.reshape(bsz, n_heads, kv_seq_len, head_dim) + Q, K, V = Q.contiguous(), K.contiguous(), V.contiguous() + A = scaled_dot_product_attention( + Q, K, V, attn_mask = attention_mask, is_causal = False + ) + A = A.transpose(1, 2).contiguous() + + attn_output = A.reshape(bsz, q_len, n_heads * head_dim) + attn_output = self.apply_o(self, attn_output) + # Optional deterministic residual dropout after attention projection + resid_attn_dropout = getattr(self, "_unsloth_resid_attn_dropout", None) + if resid_attn_dropout is not None and self.training: + attn_output = resid_attn_dropout(attn_output) + attn_weights = None + return attn_output, attn_weights, past_key_value + + +class FastPhiModel(FastLlamaModel): + @staticmethod + def pre_patch(): + if PhiAttention is None: + return + # Patch attention forward for partial RoPE support and Unsloth compute path + PhiAttention.forward = PhiAttention_fast_forward + try: + PhiSdpaAttention.forward = PhiAttention_fast_forward + PhiFlashAttention2.forward = PhiAttention_fast_forward + except Exception: + pass + # Patch CausalLM for Unsloth fastpath when compatible + try: + PhiForCausalLM.forward = CausalLM_fast_forward( + LlamaModel_fast_forward_inference + ) + PeftModelForCausalLM.forward = PeftModelForCausalLM_fast_forward + fix_prepare_inputs_for_generation(PhiForCausalLM) + except Exception: + pass + return + + @staticmethod + def post_patch(model, tokenizer): + # Ensure Phi-2 defaults for partial RoPE if missing in config + try: + if getattr(model.config, "model_type", None) == "phi": + if not hasattr(model.config, "partial_rotary_factor") and not hasattr( + model.config, "rotary_dim" + ): + # Empirically common fraction for Phi-2 partial RoPE + model.config.partial_rotary_factor = 0.4 + # Attach deterministic dropout layers if dropout > 0 for residuals + p_attn = float(getattr(model.config, "attention_dropout", 0.0)) + p_mlp = float(getattr(model.config, "hidden_dropout", 0.0)) + if p_attn > 0.0 or p_mlp > 0.0: + import os + + seed = int(os.environ.get("UNSLOTH_DROPOUT_SEED", 3407)) + for layer in model.model.layers: + if p_attn > 0.0: + # Attach to attention module so it can be used in patched attention forward + layer.self_attn._unsloth_resid_attn_dropout = ( + DeterministicDropout(p_attn, seed) + ) + if p_mlp > 0.0: + # Apply dropout to MLP outputs via a forward hook (post-MLP, pre-residual add) + layer._unsloth_resid_mlp_dropout = DeterministicDropout( + p_mlp, seed + ) + + def _mlp_hook(mod, inputs, output, _layer = layer): + if ( + _layer.training + and _layer._unsloth_resid_mlp_dropout is not None + ): + return _layer._unsloth_resid_mlp_dropout(output) + return output + + # Keep handle to prevent GC + layer._unsloth_mlp_hook = layer.mlp.register_forward_hook( + _mlp_hook + ) + except Exception: + pass + return model, tokenizer + + @staticmethod + def get_peft_model( + model, + r: int = 16, + target_modules = "all-linear", + lora_alpha: int = 16, + lora_dropout: float = 0.0, + bias: str = "none", + layers_to_transform = None, + layers_pattern = None, + use_gradient_checkpointing = True, + random_state: int = 3407, + max_seq_length: int = 2048, + use_rslora: bool = False, + modules_to_save = None, + init_lora_weights: bool = True, + loftq_config: dict = {}, + temporary_location: str = "_unsloth_temporary_saved_buffers", + **kwargs, + ): + return FastBaseModel.get_peft_model( + model = model, + r = r, + target_modules = target_modules, + lora_alpha = lora_alpha, + lora_dropout = lora_dropout, + bias = bias, + layers_to_transform = layers_to_transform, + layers_pattern = layers_pattern, + use_gradient_checkpointing = use_gradient_checkpointing, + random_state = random_state, + max_seq_length = max_seq_length, + use_rslora = use_rslora, + modules_to_save = modules_to_save, + init_lora_weights = init_lora_weights, + loftq_config = loftq_config, + temporary_location = temporary_location, + **kwargs, + ) + + @staticmethod + def from_pretrained( + model_name: str = "microsoft/Phi-2", + max_seq_length: Optional[int] = None, + dtype = None, + load_in_4bit: bool = True, + token: Optional[str] = None, + device_map: str = "sequential", + rope_scaling = None, + fix_tokenizer: bool = True, + model_patcher = None, + tokenizer_name: Optional[str] = None, + trust_remote_code: bool = False, + **kwargs, + ): + return FastLlamaModel.from_pretrained( + model_name = model_name, + max_seq_length = max_seq_length, + dtype = dtype, + load_in_4bit = load_in_4bit, + token = token, + device_map = device_map, + rope_scaling = rope_scaling, + fix_tokenizer = fix_tokenizer, + model_patcher = FastPhiModel, + tokenizer_name = tokenizer_name, + trust_remote_code = trust_remote_code, + **kwargs, + )