Bug fixes (#1255)
* Fix TRL * Update mistral.py * Patch processing_class * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Installation guide (#1165) * chore: update chat_templates.py (#1166) orginal -> original * Disable Flex Attention * Update tokenizer_utils.py * Update _utils.py * n_items * Update cross_entropy_loss.py * Fix DPO, ORPO * Update _utils.py * Update _utils.py * fix/transformers-unpack (#1180) * Fix DPO, ORPO (#1177) * Fix TRL * Update mistral.py * Patch processing_class * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Installation guide (#1165) * chore: update chat_templates.py (#1166) orginal -> original * Disable Flex Attention * Update tokenizer_utils.py * Update _utils.py * n_items * Update cross_entropy_loss.py * Fix DPO, ORPO * Update _utils.py --------- Co-authored-by: timothelaborie <97834767+timothelaborie@users.noreply.github.com> Co-authored-by: Ikko Eltociear Ashimine <eltociear@gmail.com> * Add warning for missing Unpack and KwargsForCausalLM in older Transformers versions --------- Co-authored-by: Daniel Han <danielhanchen@gmail.com> Co-authored-by: timothelaborie <97834767+timothelaborie@users.noreply.github.com> Co-authored-by: Ikko Eltociear Ashimine <eltociear@gmail.com> * Update cross_entropy_loss.py * Update _utils.py * Update _utils.py * donot upcast lm_head and embeddings to float32 (#1186) * Cleanup upcast logs (#1188) * Fix/phi-longrope (#1193) * Enhance rotary embedding handling in LlamaAttention and LongRopeRotaryEmbedding * Typo * Improve rotary embedding handling in LlamaAttention to prevent errors with short KV cache * Update llama.py * Update llama.py --------- Co-authored-by: Daniel Han <danielhanchen@gmail.com> * Update transformers * Unk token issues * Update _utils.py * Fix pad token * Update llama.py * Typo * ignored labels * Revert "ignored labels" This reverts commit9d07be077b. * More patching * Update _utils.py * Update _utils.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Feat/all tmp (#1219) * Update save.py Check whether path is in /tmp dir for Kaggle environment * Update save.py Move temporary_location to /tmp in Kaggle * Enhance Kaggle environment support in save and tokenizer utilities --------- Co-authored-by: dendarrion <37800703+dendarrion@users.noreply.github.com> Co-authored-by: Erland366 <erland.pg366@gmail.com> * Bug fixes * Update pyproject.toml * Update _utils.py * Update __init__.py * Update __init__.py * Update _utils.py * Update _utils.py * Update _utils.py * Update _utils.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Tied weights * Revert "Tied weights" This reverts commit8090b7c01a. * Tied weights * Utils * CE Loss patching * Update __init__.py * Update __init__.py * Patching * Update cross_entropy_loss.py * CE Loss * Update _utils.py * Update _utils.py * CE Loss * Update _utils.py * Update _utils.py * Layernorm * Update _utils.py * Update _utils.py * Post patch * Update _utils.py * Update llama.py * Update _utils.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * typing * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * int64 * Update _utils.py * Update cross_entropy_loss.py * constexpr * constexpr * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update _utils.py * Update _utils.py * Update _utils.py * CE * Update cross_entropy_loss.py * Update _utils.py * Update llama.py * Update _utils.py * Update rms_layernorm.py * Update rms_layernorm.py * Update rms_layernorm.py * Update rms_layernorm.py * Update rms_layernorm.py * Update rms_layernorm.py * Update utils.py * Update rms_layernorm.py * Update rms_layernorm.py * Update rms_layernorm.py * Update rms_layernorm.py * Update rms_layernorm.py * Update rms_layernorm.py * Update rms_layernorm.py * Update rms_layernorm.py * Update rms_layernorm.py * Update rms_layernorm.py * Update rms_layernorm.py * Update rms_layernorm.py * typing * Update rope_embedding.py * types * Disable compiling * Update _utils.py * Update _utils.py * Forward hook * Update _utils.py * Update llama.py * Update _utils.py * Update llama.py * Update llama.py * Update _utils.py * Update pyproject.toml * Update _utils.py * Update llama.py * CE Loss * Update cross_entropy_loss.py * Update _utils.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update llama.py * Update _utils.py * Update _utils.py * Update _utils.py * Update _utils.py * Update _utils.py --------- Co-authored-by: timothelaborie <97834767+timothelaborie@users.noreply.github.com> Co-authored-by: Ikko Eltociear Ashimine <eltociear@gmail.com> Co-authored-by: Edd <68678137+Erland366@users.noreply.github.com> Co-authored-by: Datta Nimmaturi <datta.nimmaturi@nutanix.com> Co-authored-by: dendarrion <37800703+dendarrion@users.noreply.github.com> Co-authored-by: Erland366 <erland.pg366@gmail.com>
This commit is contained in:
Daniel Han
GitHub
parent
7c684fb793
commit
4f8bf42442
1 changed files with 31 additions and 6 deletions
|
|
@ -12,7 +12,7 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
__version__ = "2024.11.1"
|
||||
__version__ = "2024.11.3"
|
||||
|
||||
__all__ = [
|
||||
"prepare_model_for_kbit_training",
|
||||
|
|
@ -69,7 +69,6 @@ from unsloth_zoo.patching_utils import (
|
|||
patch_compiling_bitsandbytes,
|
||||
patch_layernorm,
|
||||
patch_torch_compile,
|
||||
patch_regional_compilation,
|
||||
patch_model_and_tokenizer,
|
||||
)
|
||||
from unsloth_zoo.gradient_checkpointing import (
|
||||
|
|
@ -88,8 +87,9 @@ from unsloth_zoo.gradient_checkpointing import (
|
|||
# Disable some warnings which can get annoying
|
||||
warnings.filterwarnings(action = "ignore", category = UserWarning, module = "torch")
|
||||
warnings.filterwarnings(action = "ignore", category = UserWarning, module = "huggingface_hub")
|
||||
warnings.filterwarnings(action = "ignore", category = UserWarning, module = "trl")
|
||||
warnings.filterwarnings(action = "ignore", category = FutureWarning, module = "huggingface_hub")
|
||||
warnings.filterwarnings(action = "ignore", category = UserWarning, module = "trl")
|
||||
warnings.filterwarnings(action = "ignore", category = FutureWarning, module = "trl")
|
||||
warnings.filterwarnings(action = "ignore", category = FutureWarning, module = "xformers")
|
||||
warnings.filterwarnings(action = "ignore", category = RuntimeWarning, module = "subprocess")
|
||||
warnings.filterwarnings(action = "ignore", category = UserWarning, module = "transformers")
|
||||
|
|
@ -374,8 +374,9 @@ pass
|
|||
|
||||
# =============================================
|
||||
# Torch compile settings
|
||||
UNSLOTH_COMPILE_DEBUG = "UNSLOTH_COMPILE_DEBUG" in os.environ
|
||||
UNSLOTH_COMPILE_MAXIMUM = "UNSLOTH_COMPILE_MAXIMUM" in os.environ
|
||||
UNSLOTH_COMPILE_DEBUG = os.environ.get("UNSLOTH_COMPILE_DEBUG", "0") == "1"
|
||||
UNSLOTH_COMPILE_MAXIMUM = os.environ.get("UNSLOTH_COMPILE_MAXIMUM", "0") == "1"
|
||||
UNSLOTH_COMPILE_IGNORE_ERRORS = os.environ.get("UNSLOTH_COMPILE_IGNORE_ERRORS", "1") == "1"
|
||||
# Just remove max_autotune_gemm warning
|
||||
import functools
|
||||
@functools.lru_cache(None)
|
||||
|
|
@ -387,7 +388,11 @@ def is_big_gpu(index):
|
|||
return True
|
||||
import torch._inductor.utils
|
||||
torch._inductor.utils.is_big_gpu = is_big_gpu
|
||||
patch_torch_compile(debug = UNSLOTH_COMPILE_DEBUG, O3 = UNSLOTH_COMPILE_MAXIMUM)
|
||||
patch_torch_compile(
|
||||
debug = UNSLOTH_COMPILE_DEBUG,
|
||||
O3 = UNSLOTH_COMPILE_MAXIMUM,
|
||||
ignore_errors = UNSLOTH_COMPILE_IGNORE_ERRORS,
|
||||
)
|
||||
|
||||
torch_compile_options = {
|
||||
"epilogue_fusion" : True,
|
||||
|
|
@ -408,6 +413,26 @@ accelerate.utils.TorchDynamoPlugin.to_kwargs = torch_compile_kwargs
|
|||
accelerate.accelerator.TorchDynamoPlugin.to_kwargs = torch_compile_kwargs
|
||||
del accelerate
|
||||
|
||||
def patch_regional_compilation():
|
||||
# Regional torch 2.5 Recompilation - weirdly very slow??
|
||||
if torch.nn.ModuleList.__name__ == "UnslothModuleList": return
|
||||
# Only works for torch 2.5
|
||||
if Version(torch.__version__) < Version("2.5.0"): return
|
||||
|
||||
old_module_list = torch.nn.ModuleList
|
||||
os.environ["UNSLOTH_PATCHED"] = "1"
|
||||
|
||||
def UnslothModuleList(*args, **kwargs):
|
||||
if len(args) == 1 and len(kwargs) == 0 and type(args[0]) is list:
|
||||
args = [old_module_list([torch.compile(x, dynamic = True, options = torch_compile_options, fullgraph = False) for x in args[0]])]
|
||||
return old_module_list(*args, **kwargs)
|
||||
pass
|
||||
UnslothModuleList.__doc__ = old_module_list.__doc__
|
||||
|
||||
torch.nn.ModuleList = UnslothModuleList
|
||||
return
|
||||
pass
|
||||
|
||||
# =============================================
|
||||
|
||||
def prepare_model_for_kbit_training(
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue