From 5c7cb059237d5245b290f066de1dfdd8ed9d918e Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 25 Sep 2024 12:42:24 -0700 Subject: [PATCH] Update llama.py --- unsloth/models/llama.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index f5dc02704e..bae6d5b80d 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -1387,7 +1387,8 @@ def _wrap_fast_inference(generate, device_type, dtype, model): # For newer HF if SUPPORTS_LLAMA32: - kwargs["cache_implementation"] = "hybrid" + # kwargs["cache_implementation"] = "hybrid" + pass else: kwargs["cache_implementation"] = "dynamic" # For num_logits_to_keep