Update llama.py

2024-09-25 12:42:24 -07:00 · 2024-09-25 12:42:24 -07:00 · 3cc9c2410e
commit 3cc9c2410e
parent 0c07b760de
1 changed files with 2 additions and 1 deletions
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@ -1387,7 +1387,8 @@ def _wrap_fast_inference(generate, device_type, dtype, model):

        # For newer HF
        if SUPPORTS_LLAMA32:
-            kwargs["cache_implementation"] = "hybrid"
+            # kwargs["cache_implementation"] = "hybrid"
+            pass
        else:
            kwargs["cache_implementation"] = "dynamic"
        # For num_logits_to_keep