ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fp8 cache

Browse source
This commit is contained in:
Daniel Han 2025-02-05 20:00:02 -08:00
commit f73d7789a4
2 changed files with 2 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

2
unsloth/models/llama.py
View file

@ -1637,7 +1637,7 @@ class FastLlamaModel:
fast_inference = False, # uses vLLM
gpu_memory_utilization = 0.5,
float8_kv_cache = True,
float8_kv_cache = False,
random_state = 3407,
max_lora_rank = 16,
disable_log_stats = False,

2
unsloth/models/loader.py
View file

@ -76,7 +76,7 @@ class FastLanguageModel(FastLlamaModel):
fast_inference = False, # uses vLLM
gpu_memory_utilization = 0.5,
float8_kv_cache = True,
float8_kv_cache = False,
random_state = 3407,
max_lora_rank = 16,
disable_log_stats = False,