From d1db2659657dc451f77bc2d802d1943b16ef0944 Mon Sep 17 00:00:00 2001 From: Mukkesh Ganesh Date: Sun, 16 Mar 2025 15:19:14 -0700 Subject: [PATCH] bug fix #2008 (#2039) --- unsloth/models/llama.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index a96b435b58..b2fbf0d07f 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -1825,7 +1825,7 @@ class FastLlamaModel: # Convert to HF format _, quant_state_dict = get_vllm_state_dict(llm, config = model_config) - model = convert_vllm_to_huggingface(quant_state_dict, model_config, dtype) + model = convert_vllm_to_huggingface(quant_state_dict, model_config, dtype, bnb_config) model.vllm_engine = llm model.fast_generate = model.vllm_engine.generate model.fast_generate_batches = functools.partial(generate_batches, model.vllm_engine)