ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

bug fix #2008 (#2039)

Browse source
This commit is contained in:
Mukkesh Ganesh 2025-03-16 15:19:14 -07:00 • committed by GitHub
commit d1db265965
1 changed files with 1 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

2
unsloth/models/llama.py
View file

@ -1825,7 +1825,7 @@ class FastLlamaModel:
# Convert to HF format
_, quant_state_dict = get_vllm_state_dict(llm, config = model_config)
model = convert_vllm_to_huggingface(quant_state_dict, model_config, dtype)
model = convert_vllm_to_huggingface(quant_state_dict, model_config, dtype, bnb_config)
model.vllm_engine = llm
model.fast_generate = model.vllm_engine.generate
model.fast_generate_batches = functools.partial(generate_batches, model.vllm_engine)