From 1e77ab25b6932fac39b375e667182f13e79233fb Mon Sep 17 00:00:00 2001 From: Daniel Han-Chen Date: Wed, 7 Feb 2024 17:43:51 +1100 Subject: [PATCH] Update llama.py --- unsloth/models/llama.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index 08da7adc3b..4734233405 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -618,7 +618,15 @@ pass # https://github.com/huggingface/transformers/blob/main/src/transformers/models/llama/modeling_llama.py#L825 -@torch.compile(mode = "max-autotune-no-cudagraphs", options = {"trace.enabled" : True, "trace.graph_diagram" : True,}, dynamic = True,) +@torch.compile(options = { + "epilogue_fusion" : True, + "max_autotune" : True, + "fallback_random" : False, + "shape_padding" : True, + "triton.cudagraphs" : False, + "trace.enabled" : True, + "trace.graph_diagram" : True, +}, dynamic = True,) @torch.inference_mode def LlamaModel_fast_forward_inference( self,