From f73d7789a41a57b34beca1d2a7a18eaaec273504 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 5 Feb 2025 20:00:02 -0800 Subject: [PATCH] Fp8 cache --- unsloth/models/llama.py | 2 +- unsloth/models/loader.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index ab90d2cbb1..a337472a3e 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -1637,7 +1637,7 @@ class FastLlamaModel: fast_inference = False, # uses vLLM gpu_memory_utilization = 0.5, - float8_kv_cache = True, + float8_kv_cache = False, random_state = 3407, max_lora_rank = 16, disable_log_stats = False, diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index 144863b8da..ad312e0040 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -76,7 +76,7 @@ class FastLanguageModel(FastLlamaModel): fast_inference = False, # uses vLLM gpu_memory_utilization = 0.5, - float8_kv_cache = True, + float8_kv_cache = False, random_state = 3407, max_lora_rank = 16, disable_log_stats = False,