From cb06d8576c86bc18a6d216a2c31667858e4cce25 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 14 Mar 2025 04:02:41 -0700 Subject: [PATCH] Update llama.py --- unsloth/models/llama.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index 38801d23db..0f2996d212 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -2659,11 +2659,11 @@ class FastLlamaModel: pass # Patch for fast inference - vllm_engine = getattr(model, "vllm_engine", None) + vllm_engine = getattr(model.model, "vllm_engine", None) if vllm_engine is not None: - model.vllm_engine = vllm_engine - model.fast_generate = vllm_fast_generate - model.fast_generate_batches = vllm_fast_generate_batches + model.vllm_engine = model.model.vllm_engine + model.fast_generate = model.model.vllm_fast_generate + model.fast_generate_batches = model.model.vllm_fast_generate_batches # Also saving and loading LoRA from unsloth_zoo.vllm_utils import save_lora, load_lora