diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 3b4a351e09..234b90e578 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -2687,32 +2687,17 @@ def make_fast_generate_wrapper(original_generate): def _fast_generate_wrapper(*args, **kwargs): # Check for vLLM-specific arguments if "sampling_params" in kwargs: - if DEVICE_TYPE == "hip": - # Allow GRPO notebooks to run on AMD without vLLM - print( - "Unsloth: `sampling_params` ignored because fast inference is " - "disabled on AMD." - ) - kwargs.pop("sampling_params", None) - else: - raise ValueError( - "Unsloth: `sampling_params` is only supported when `fast_inference=True` (vLLM). " - "Since `fast_inference=False`, use HuggingFace generate arguments instead:\n" - " model.fast_generate(**tokens.to('cuda'), max_new_tokens=64, temperature=1.0, top_p=0.95)" - ) + raise ValueError( + "Unsloth: `sampling_params` is only supported when `fast_inference=True` (vLLM). " + "Since `fast_inference=False`, use HuggingFace generate arguments instead:\n" + " model.fast_generate(**tokens.to('cuda'), max_new_tokens=64, temperature=1.0, top_p=0.95)" + ) if "lora_request" in kwargs: - if DEVICE_TYPE == "hip": - print( - "Unsloth: `lora_request` ignored because fast inference is " - "disabled on AMD." - ) - kwargs.pop("lora_request", None) - else: - raise ValueError( - "Unsloth: `lora_request` is only supported when `fast_inference=True` (vLLM). " - "Since `fast_inference=False`, LoRA weights are already merged into the model." - ) + raise ValueError( + "Unsloth: `lora_request` is only supported when `fast_inference=True` (vLLM). " + "Since `fast_inference=False`, LoRA weights are already merged into the model." + ) # Check if first positional argument is a string or list of strings if len(args) > 0: @@ -2726,40 +2711,21 @@ def make_fast_generate_wrapper(original_generate): is_string_input = True if is_string_input: - if DEVICE_TYPE == "hip": - model = getattr(original_generate, "__self__", None) - tokenizer = getattr(model, "_saved_temp_tokenizer", None) - if tokenizer is None: - raise ValueError( - "Unsloth: Passing text strings to `fast_generate` on AMD " - "requires a tokenizer attached to the model." - ) - texts = ( - [first_arg] if isinstance(first_arg, str) else list(first_arg) - ) - tokens = tokenizer( - texts, - return_tensors = "pt", - padding = True, - ) - tokens = tokens.to(model.device) - return original_generate(**tokens, **kwargs) - else: - raise ValueError( - "Unsloth: Passing text strings to `fast_generate` is only supported " - "when `fast_inference=True` (vLLM). Since `fast_inference=False`, you must " - "tokenize the input first:\n\n" - " messages = tokenizer.apply_chat_template(\n" - ' [{"role": "user", "content": "Your prompt here"}],\n' - " tokenize=True, add_generation_prompt=True,\n" - ' return_tensors="pt", return_dict=True\n' - " )\n" - " output = model.fast_generate(\n" - " **messages.to('cuda'),\n" - " max_new_tokens=64,\n" - " temperature=1.0,\n" - " )" - ) + raise ValueError( + "Unsloth: Passing text strings to `fast_generate` is only supported " + "when `fast_inference=True` (vLLM). Since `fast_inference=False`, you must " + "tokenize the input first:\n\n" + " messages = tokenizer.apply_chat_template(\n" + ' [{"role": "user", "content": "Your prompt here"}],\n' + " tokenize=True, add_generation_prompt=True,\n" + ' return_tensors="pt", return_dict=True\n' + " )\n" + " output = model.fast_generate(\n" + " **messages.to('cuda'),\n" + " max_new_tokens=64,\n" + " temperature=1.0,\n" + " )" + ) # Call original generate return original_generate(*args, **kwargs) diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index 5d3d7bb62c..39ffc35ceb 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -246,18 +246,10 @@ class FastLanguageModel(FastLlamaModel): if fast_inference: if importlib.util.find_spec("vllm") is None: - if DEVICE_TYPE == "hip": - print( - "Unsloth: vLLM not installed on AMD; falling back to native " - "inference. Install vLLM or set `fast_inference=False` to " - "silence this warning." - ) - fast_inference = False - else: - raise ImportError( - "Unsloth: Please install vLLM before enabling `fast_inference`!\n" - "You can do this in a terminal via `pip install vllm`" - ) + raise ImportError( + "Unsloth: Please install vLLM before enabling `fast_inference`!\n" + "You can do this in a terminal via `pip install vllm`" + ) if DEVICE_TYPE_TORCH == "cuda": for i in range(DEVICE_COUNT): # [TODO] DGX Spark vLLM breaks @@ -272,16 +264,9 @@ class FastLanguageModel(FastLlamaModel): # [TODO] For now fast_inference only works with fast_inference ie vLLM if load_in_fp8 != False: if not fast_inference: - if DEVICE_TYPE == "hip": - print( - "Unsloth: `load_in_fp8` requires fast inference. Disabling " - "FP8 on AMD for now." - ) - load_in_fp8 = False - else: - raise NotImplementedError( - "Unsloth: set `fast_inference = True` when doing `load_in_fp8`." - ) + raise NotImplementedError( + "Unsloth: set `fast_inference = True` when doing `load_in_fp8`." + ) # Check if 4bit is allowed specifically for AMD if not ALLOW_BITSANDBYTES and not use_exact_model_name: if load_in_4bit or load_in_8bit or model_name.lower().endswith("-bnb-4bit"): @@ -928,18 +913,10 @@ class FastModel(FastBaseModel): if fast_inference: if importlib.util.find_spec("vllm") is None: - if DEVICE_TYPE == "hip": - print( - "Unsloth: vLLM not installed on AMD; falling back to native " - "inference. Install vLLM or set `fast_inference=False` to " - "silence this warning." - ) - fast_inference = False - else: - raise ImportError( - "Unsloth: Please install vLLM before enabling `fast_inference`!\n" - "You can do this in a terminal via `pip install vllm`" - ) + raise ImportError( + "Unsloth: Please install vLLM before enabling `fast_inference`!\n" + "You can do this in a terminal via `pip install vllm`" + ) if DEVICE_TYPE_TORCH == "cuda": for i in range(DEVICE_COUNT): # [TODO] DGX Spark vLLM breaks @@ -954,16 +931,9 @@ class FastModel(FastBaseModel): # [TODO] For now fast_inference only works with fast_inference ie vLLM if load_in_fp8 != False: if not fast_inference: - if DEVICE_TYPE == "hip": - print( - "Unsloth: `load_in_fp8` requires fast inference. Disabling " - "FP8 on AMD for now." - ) - load_in_fp8 = False - else: - raise NotImplementedError( - "Unsloth: set `fast_inference = True` when doing `load_in_fp8`." - ) + raise NotImplementedError( + "Unsloth: set `fast_inference = True` when doing `load_in_fp8`." + ) # Find FP8, BnB 4bit, other mapped names old_model_name = model_name