ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

ROCm: trim unintended fast-inference fallback behaviors

Browse source
This commit is contained in:
Daniel Han-Chen 2026-02-11 06:48:37 +00:00
commit 8a117da4b6
2 changed files with 38 additions and 102 deletions
Download patch file Download diff file Expand all files Collapse all files

82
unsloth/models/_utils.py
View file

@ -2687,32 +2687,17 @@ def make_fast_generate_wrapper(original_generate):
def _fast_generate_wrapper(*args, **kwargs):
# Check for vLLM-specific arguments
if "sampling_params" in kwargs:
if DEVICE_TYPE == "hip":
# Allow GRPO notebooks to run on AMD without vLLM
print(
"Unsloth: `sampling_params` ignored because fast inference is "
"disabled on AMD."
)
kwargs.pop("sampling_params", None)
else:
raise ValueError(
"Unsloth: `sampling_params` is only supported when `fast_inference=True` (vLLM). "
"Since `fast_inference=False`, use HuggingFace generate arguments instead:\n"
" model.fast_generate(**tokens.to('cuda'), max_new_tokens=64, temperature=1.0, top_p=0.95)"
)
raise ValueError(
"Unsloth: `sampling_params` is only supported when `fast_inference=True` (vLLM). "
"Since `fast_inference=False`, use HuggingFace generate arguments instead:\n"
" model.fast_generate(**tokens.to('cuda'), max_new_tokens=64, temperature=1.0, top_p=0.95)"
)
if "lora_request" in kwargs:
if DEVICE_TYPE == "hip":
print(
"Unsloth: `lora_request` ignored because fast inference is "
"disabled on AMD."
)
kwargs.pop("lora_request", None)
else:
raise ValueError(
"Unsloth: `lora_request` is only supported when `fast_inference=True` (vLLM). "
"Since `fast_inference=False`, LoRA weights are already merged into the model."
)
raise ValueError(
"Unsloth: `lora_request` is only supported when `fast_inference=True` (vLLM). "
"Since `fast_inference=False`, LoRA weights are already merged into the model."
)
# Check if first positional argument is a string or list of strings
if len(args) > 0:
@ -2726,40 +2711,21 @@ def make_fast_generate_wrapper(original_generate):
is_string_input = True
if is_string_input:
if DEVICE_TYPE == "hip":
model = getattr(original_generate, "__self__", None)
tokenizer = getattr(model, "_saved_temp_tokenizer", None)
if tokenizer is None:
raise ValueError(
"Unsloth: Passing text strings to `fast_generate` on AMD "
"requires a tokenizer attached to the model."
)
texts = (
[first_arg] if isinstance(first_arg, str) else list(first_arg)
)
tokens = tokenizer(
texts,
return_tensors = "pt",
padding = True,
)
tokens = tokens.to(model.device)
return original_generate(**tokens, **kwargs)
else:
raise ValueError(
"Unsloth: Passing text strings to `fast_generate` is only supported "
"when `fast_inference=True` (vLLM). Since `fast_inference=False`, you must "
"tokenize the input first:\n\n"
" messages = tokenizer.apply_chat_template(\n"
' [{"role": "user", "content": "Your prompt here"}],\n'
" tokenize=True, add_generation_prompt=True,\n"
' return_tensors="pt", return_dict=True\n'
" )\n"
" output = model.fast_generate(\n"
" **messages.to('cuda'),\n"
" max_new_tokens=64,\n"
" temperature=1.0,\n"
" )"
)
raise ValueError(
"Unsloth: Passing text strings to `fast_generate` is only supported "
"when `fast_inference=True` (vLLM). Since `fast_inference=False`, you must "
"tokenize the input first:\n\n"
" messages = tokenizer.apply_chat_template(\n"
' [{"role": "user", "content": "Your prompt here"}],\n'
" tokenize=True, add_generation_prompt=True,\n"
' return_tensors="pt", return_dict=True\n'
" )\n"
" output = model.fast_generate(\n"
" **messages.to('cuda'),\n"
" max_new_tokens=64,\n"
" temperature=1.0,\n"
" )"
)
# Call original generate
return original_generate(*args, **kwargs)

58
unsloth/models/loader.py
View file

@ -246,18 +246,10 @@ class FastLanguageModel(FastLlamaModel):
if fast_inference:
if importlib.util.find_spec("vllm") is None:
if DEVICE_TYPE == "hip":
print(
"Unsloth: vLLM not installed on AMD; falling back to native "
"inference. Install vLLM or set `fast_inference=False` to "
"silence this warning."
)
fast_inference = False
else:
raise ImportError(
"Unsloth: Please install vLLM before enabling `fast_inference`!\n"
"You can do this in a terminal via `pip install vllm`"
)
raise ImportError(
"Unsloth: Please install vLLM before enabling `fast_inference`!\n"
"You can do this in a terminal via `pip install vllm`"
)
if DEVICE_TYPE_TORCH == "cuda":
for i in range(DEVICE_COUNT):
# [TODO] DGX Spark vLLM breaks
@ -272,16 +264,9 @@ class FastLanguageModel(FastLlamaModel):
# [TODO] For now fast_inference only works with fast_inference ie vLLM
if load_in_fp8 != False:
if not fast_inference:
if DEVICE_TYPE == "hip":
print(
"Unsloth: `load_in_fp8` requires fast inference. Disabling "
"FP8 on AMD for now."
)
load_in_fp8 = False
else:
raise NotImplementedError(
"Unsloth: set `fast_inference = True` when doing `load_in_fp8`."
)
raise NotImplementedError(
"Unsloth: set `fast_inference = True` when doing `load_in_fp8`."
)
# Check if 4bit is allowed specifically for AMD
if not ALLOW_BITSANDBYTES and not use_exact_model_name:
if load_in_4bit or load_in_8bit or model_name.lower().endswith("-bnb-4bit"):
@ -928,18 +913,10 @@ class FastModel(FastBaseModel):
if fast_inference:
if importlib.util.find_spec("vllm") is None:
if DEVICE_TYPE == "hip":
print(
"Unsloth: vLLM not installed on AMD; falling back to native "
"inference. Install vLLM or set `fast_inference=False` to "
"silence this warning."
)
fast_inference = False
else:
raise ImportError(
"Unsloth: Please install vLLM before enabling `fast_inference`!\n"
"You can do this in a terminal via `pip install vllm`"
)
raise ImportError(
"Unsloth: Please install vLLM before enabling `fast_inference`!\n"
"You can do this in a terminal via `pip install vllm`"
)
if DEVICE_TYPE_TORCH == "cuda":
for i in range(DEVICE_COUNT):
# [TODO] DGX Spark vLLM breaks
@ -954,16 +931,9 @@ class FastModel(FastBaseModel):
# [TODO] For now fast_inference only works with fast_inference ie vLLM
if load_in_fp8 != False:
if not fast_inference:
if DEVICE_TYPE == "hip":
print(
"Unsloth: `load_in_fp8` requires fast inference. Disabling "
"FP8 on AMD for now."
)
load_in_fp8 = False
else:
raise NotImplementedError(
"Unsloth: set `fast_inference = True` when doing `load_in_fp8`."
)
raise NotImplementedError(
"Unsloth: set `fast_inference = True` when doing `load_in_fp8`."
)
# Find FP8, BnB 4bit, other mapped names
old_model_name = model_name