Fix merges (#1079)
* Layernorm * Update layernorm.py * Update layernorm.py * Update layernorm.py * Update layernorm.py * Update layernorm.py * Update layernorm.py * Patch layernorm * Update layernorm.py * RMS Layernorm * Update rms_layernorm.py * Causal LM * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update layernorm.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update _utils.py * Update _utils.py * Llama 3.2 * Update _utils.py * Update _utils.py * Update _utils.py * Update llama.py * Update vision.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update loader.py * Update loader.py * Update loader.py * Dependencies * Update pyproject.toml * Update _utils.py
This commit is contained in:
Daniel Han
GitHub
parent
e694d3541f
commit
ccfb9b68ff
5 changed files with 68 additions and 62 deletions
|
|
@ -35,15 +35,15 @@ exclude = ["images*"]
|
|||
huggingface = [
|
||||
"packaging",
|
||||
"tyro",
|
||||
"transformers>=4.45.1",
|
||||
"transformers<4.45.0",
|
||||
"datasets>=2.16.0",
|
||||
"sentencepiece>=0.2.0",
|
||||
"tqdm",
|
||||
"psutil",
|
||||
"wheel>=0.42.0",
|
||||
"numpy",
|
||||
"accelerate>=0.26.1",
|
||||
"trl>=0.7.9,!=0.9.0,!=0.9.1,!=0.9.2,!=0.9.3",
|
||||
"accelerate>=0.34.1",
|
||||
"trl>=0.7.9,!=0.9.0,!=0.9.1,!=0.9.2,!=0.9.3,<=0.11.1",
|
||||
"peft>=0.7.1,!=0.11.0",
|
||||
"protobuf<4.0.0",
|
||||
"huggingface_hub",
|
||||
|
|
@ -212,7 +212,7 @@ colab-ampere-torch220 = [
|
|||
colab-new = [
|
||||
"packaging",
|
||||
"tyro",
|
||||
"transformers>=4.45.1",
|
||||
"transformers<4.45.0",
|
||||
"datasets>=2.16.0",
|
||||
"sentencepiece>=0.2.0",
|
||||
"tqdm",
|
||||
|
|
@ -224,8 +224,8 @@ colab-new = [
|
|||
"hf_transfer",
|
||||
]
|
||||
colab-no-deps = [
|
||||
"accelerate>=0.26.1",
|
||||
"trl>=0.7.9,!=0.9.0,!=0.9.1,!=0.9.2,!=0.9.3",
|
||||
"accelerate>=0.34.1",
|
||||
"trl>=0.7.9,!=0.9.0,!=0.9.1,!=0.9.2,!=0.9.3,<=0.11.1",
|
||||
"peft>=0.7.1",
|
||||
"xformers<0.0.27",
|
||||
"bitsandbytes>=0.43.3",
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
__version__ = "2024.9.post3"
|
||||
__version__ = "2024.9.post4"
|
||||
|
||||
__all__ = [
|
||||
"prepare_model_for_kbit_training",
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ from peft import PeftConfig, PeftModel
|
|||
from .mapper import INT_TO_FLOAT_MAPPER, FLOAT_TO_INT_MAPPER, MAP_TO_UNSLOTH_16bit
|
||||
import os
|
||||
from huggingface_hub.utils._token import get_token
|
||||
from huggingface_hub import HfFileSystem
|
||||
|
||||
# https://github.com/huggingface/transformers/pull/26037 allows 4 bit loading!
|
||||
from packaging.version import Version
|
||||
|
|
@ -191,14 +192,29 @@ class FastLanguageModel(FastLlamaModel):
|
|||
is_peft = False
|
||||
pass
|
||||
|
||||
# Cannot be both!
|
||||
if (is_model and is_peft) and not SUPPORTS_LLAMA32:
|
||||
# Both config.json and adapter_config.json should not exist!
|
||||
|
||||
# Old transformers versions check
|
||||
both_exist = (is_model and is_peft) and not SUPPORTS_LLAMA32
|
||||
|
||||
if SUPPORTS_LLAMA32:
|
||||
# New transformers need to check manually.
|
||||
files = HfFileSystem(token = token).glob(os.path.join(model_name, "*.json"))
|
||||
files = (os.path.split(x)[-1] for x in files)
|
||||
if sum(x == "adapter_config.json" or x == "config.json" for x in files) >= 2:
|
||||
both_exist = True
|
||||
pass
|
||||
pass
|
||||
|
||||
# Error out if both LoRA and normal model config exists.
|
||||
if both_exist:
|
||||
raise RuntimeError(
|
||||
"Unsloth: Your repo has a LoRA adapter and a base model.\n"\
|
||||
"You have 2 files `config.json` and `adapter_config.json`.\n"\
|
||||
"We must only allow one config file.\n"\
|
||||
"Please separate the LoRA and base models to 2 repos."
|
||||
)
|
||||
|
||||
elif not is_model and not is_peft:
|
||||
error = autoconfig_error or peft_error
|
||||
# Old transformers version
|
||||
|
|
|
|||
|
|
@ -18,8 +18,14 @@ from ..kernels import patch_rms_layernorm, unpatch_rms_layernorm
|
|||
from ..kernels import patch_llama_for_causal_lm, unpatch_llama_for_causal_lm
|
||||
from ._utils import patch_gradient_checkpointing
|
||||
|
||||
from transformers import AutoProcessor, AutoModelForVision2Seq
|
||||
|
||||
from transformers import AutoProcessor
|
||||
try:
|
||||
from transformers import MllamaForConditionalGeneration
|
||||
except:
|
||||
raise ImportError(
|
||||
"Unsloth: Please update your transformers version to 4.46.0 for Llama 3.2 support!"
|
||||
)
|
||||
pass
|
||||
|
||||
class FastVisionModel:
|
||||
|
||||
|
|
@ -56,7 +62,6 @@ class FastVisionModel:
|
|||
)
|
||||
pass
|
||||
if token is None: token = get_token()
|
||||
if model_patcher is None: model_patcher = FastLlamaModel
|
||||
SUPPORTS_BFLOAT16 = is_bfloat16_supported()
|
||||
gpu_stats = torch.cuda.get_device_properties(0)
|
||||
max_memory = round(gpu_stats.total_memory / 1024 / 1024 / 1024, 3)
|
||||
|
|
@ -87,48 +92,6 @@ class FastVisionModel:
|
|||
|
||||
assert(dtype == torch.float16 or dtype == torch.bfloat16 or dtype == torch.float32)
|
||||
|
||||
# RoPE Scaling
|
||||
model_config = AutoConfig.from_pretrained(model_name, token = token)
|
||||
model_max_seq_length = model_config.max_position_embeddings
|
||||
|
||||
# Check if RoPE Scaling is even allowed
|
||||
model_function = MODEL_FOR_CAUSAL_LM_MAPPING[model_config.__class__]
|
||||
has_rope_scaling = False
|
||||
try:
|
||||
with open(inspect.getfile(model_function), "r") as file:
|
||||
has_rope_scaling = "self.config.rope_scaling" in file.read()
|
||||
except: pass
|
||||
has_rope_scaling = True
|
||||
|
||||
# If max_seq_length is not specified, use maximum fron config
|
||||
if max_seq_length is None:
|
||||
max_seq_length = model_max_seq_length
|
||||
pass
|
||||
|
||||
if (rope_scaling is None) and (max_seq_length > model_max_seq_length):
|
||||
|
||||
rope_scaling = max_seq_length / model_max_seq_length
|
||||
|
||||
logger.warning_once(
|
||||
f"Unsloth: {model_name} can only handle sequence lengths of at most "\
|
||||
f"{model_max_seq_length}.\nBut with kaiokendev's RoPE scaling of "\
|
||||
f"{round(rope_scaling, 3)}, it can be magically be extended to "\
|
||||
f"{max_seq_length}!"
|
||||
)
|
||||
|
||||
# Warn RoPE scaling isn't allowed
|
||||
if not has_rope_scaling:
|
||||
raise RuntimeError(
|
||||
"However, {model_name} doesn't support RoPE Scaling!\n"\
|
||||
"Please file a feature request at https://github.com/unslothai/unsloth."
|
||||
)
|
||||
pass
|
||||
|
||||
rope_scaling = {"type": "linear", "factor": rope_scaling,}
|
||||
|
||||
# Add to kwargs
|
||||
kwargs["rope_scaling"] = rope_scaling
|
||||
pass
|
||||
# We currently only support NVIDIA GPUs - AMD / Intel is a work in progress!
|
||||
pre_check = check_nvidia()
|
||||
|
||||
|
|
@ -142,16 +105,11 @@ class FastVisionModel:
|
|||
)
|
||||
pass
|
||||
|
||||
# https://huggingface.co/togethercomputer/LLaMA-2-7B-32K/discussions/12
|
||||
# RoPE Scaling's max_position_embeddings must be updated
|
||||
max_position_embeddings = max(max_seq_length, model_max_seq_length)
|
||||
kwargs.pop("attn_implementation", None); # No need since we auto call it
|
||||
|
||||
# Cannot be None, since HF now checks for the config
|
||||
if load_in_4bit: kwargs["quantization_config"] = bnb_config
|
||||
|
||||
self.pre_patch()
|
||||
model = AutoModelForVision2Seq.from_pretrained(
|
||||
model = MllamaForConditionalGeneration.from_pretrained(
|
||||
model_name,
|
||||
device_map = device_map,
|
||||
torch_dtype = dtype,
|
||||
|
|
@ -159,7 +117,7 @@ class FastVisionModel:
|
|||
token = token,
|
||||
max_position_embeddings = max_position_embeddings,
|
||||
trust_remote_code = trust_remote_code,
|
||||
attn_implementation = "eager",
|
||||
attn_implementation = "sdpa",
|
||||
**kwargs,
|
||||
)
|
||||
self.post_unpatch()
|
||||
|
|
|
|||
|
|
@ -1135,7 +1135,39 @@ from inspect import getsource
|
|||
import trl.trainer.sft_trainer
|
||||
from trl.trainer.sft_trainer import *
|
||||
from transformers.trainer import *
|
||||
from trl.trainer.sft_trainer import neftune_post_forward_hook
|
||||
try:
|
||||
from trl.trainer.sft_trainer import neftune_post_forward_hook
|
||||
except:
|
||||
def neftune_post_forward_hook(module, input, output):
|
||||
"""
|
||||
Implements the NEFTune forward pass for the model using forward hooks. Note this works only for
|
||||
torch.nn.Embedding layers. This method is slightly adapted from the original source code
|
||||
that can be found here: https://github.com/neelsjain/NEFTune
|
||||
|
||||
Simply add it to your model as follows:
|
||||
```python
|
||||
model = ...
|
||||
model.embed_tokens.neftune_noise_alpha = 0.1
|
||||
model.embed_tokens.register_forward_hook(neftune_post_forward_hook)
|
||||
```
|
||||
|
||||
Args:
|
||||
module (`torch.nn.Module`):
|
||||
The embedding module where the hook is attached. Note that you need to set
|
||||
`module.neftune_noise_alpha` to the desired noise alpha value.
|
||||
input (`torch.Tensor`):
|
||||
The input tensor to the model.
|
||||
output (`torch.Tensor`):
|
||||
The output tensor of the model (i.e. the embeddings).
|
||||
"""
|
||||
if module.training:
|
||||
dims = torch.tensor(output.size(1) * output.size(2))
|
||||
mag_norm = module.neftune_noise_alpha / torch.sqrt(dims)
|
||||
output = output + torch.zeros_like(output).uniform_(-mag_norm, mag_norm)
|
||||
return output
|
||||
pass
|
||||
pass
|
||||
|
||||
|
||||
def patch_sft_trainer_tokenizer():
|
||||
"""
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue