From 25c6e1d8a77132bb662263e697f69266261ffef7 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 25 Sep 2024 14:32:29 -0700 Subject: [PATCH] Update llama.py --- unsloth/models/llama.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index bae6d5b80d..2524ec1312 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -1386,11 +1386,7 @@ def _wrap_fast_inference(generate, device_type, dtype, model): pass # For newer HF - if SUPPORTS_LLAMA32: - # kwargs["cache_implementation"] = "hybrid" - pass - else: - kwargs["cache_implementation"] = "dynamic" + kwargs["cache_implementation"] = "dynamic" # For num_logits_to_keep kwargs["num_logits_to_keep"] = 1