From 0472fba0a6b38af43650a166e30ebd2391d07cce Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 25 Sep 2024 22:12:21 -0700 Subject: [PATCH] Update llama.py --- unsloth/models/llama.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index bae6d5b80d..2524ec1312 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -1386,11 +1386,7 @@ def _wrap_fast_inference(generate, device_type, dtype, model): pass # For newer HF - if SUPPORTS_LLAMA32: - # kwargs["cache_implementation"] = "hybrid" - pass - else: - kwargs["cache_implementation"] = "dynamic" + kwargs["cache_implementation"] = "dynamic" # For num_logits_to_keep kwargs["num_logits_to_keep"] = 1