Bug fixes (#2113)
* Update rl.py * Update rl.py * Update _utils.py * Update __init__.py * Update _utils.py * Version * versioning * Update _utils.py * Update llama.py * Update llama.py * Bug fixes * FastModel * __doc__ * Update vision.py * Update loader.py * Update loader.py * Update loader.py * version * move use_modelscope to _utils (#1938) * move use_modelscope to _utils * Update _utils.py * Update loader.py --------- Co-authored-by: Daniel Han <danielhanchen@gmail.com> * Don't use revision when loading model_config and is_peft=True (#1949) * More syntax warnings (#1944) * move use_modelscope to _utils * fix * Update _utils.py * Update loader.py --------- Co-authored-by: Daniel Han <danielhanchen@gmail.com> * Update loader.py * Full finetuning and other fixes * UNSLOTH_ENABLE_FULL_FINETUNING * Update loader.py * Update loader.py * Update loader.py * Update vision.py * Update vision.py * full finetuning * Update loader.py * Update loader.py * Update loader.py * Update _utils.py * max_seq_length * Update rl.py * Update rl.py * Update rl.py * Update pyproject.toml * AutoModelForImageTextToText * Update mapper.py * Update pyproject.toml * Update _utils.py * Update _utils.py * Update _utils.py * Batch samples * Update loader.py * Update loader.py * Update loader.py * Update loader.py * Update _utils.py * Update loader.py * Update vision.py * Update loader.py * Update vision.py * Update vision.py * Update vision.py * Update mapper.py * Update vision.py * Temporary patches * Update loader.py * model names * Gemma 3 chat template * Bug fixes * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update llama.py * Update llama.py * Update rl.py * Update chat_templates.py * Update chat_templates.py * Update vision.py * Update vision.py * Update vision.py * Update loader.py * Update vision.py * Update vision.py * Revert * Update _utils.py * forced precision * Autocast * Update vision.py * Update vision.py * Update rl.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update rl.py * vLLM fixes * constexpr * Update vision.py * Update vision.py * Update vision.py * Update rl.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update _utils.py * Update _utils.py * Update _utils.py * Update _utils.py * Update save.py * New models * Triton windows update (#1976) * Update pyproject.toml * Update README.md * Update RMS LayerNorm implementation, and list compr. change in chat templates (#1974) * Update RMS LayerNorm implementation with optimizations and testing suite * perf: optimize list comprehension in get_ollama_eos_tokens * Update Zoo * Update llama.py * Update llama.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update rl_replacements.py * Update vision.py * grpo fix * Update rl_replacements.py * Update vision.py * Update rl_replacements.py * Update vision.py * Update mapper.py * Update vision.py * Update vision.py * Update loader.py * Update vision.py * Update save.py * Update save.py * Update save.py * Update rl.py * Update _utils.py * Version * Update pyproject.toml * Update llama.py * Update llama.py * bug fix #2008 (#2039) * fix (#2051) * Update loader.py * Update pyproject.toml * Update pyproject.toml * Update vision.py * more prints * Update loader.py * LoRA 16bit fix * Update vision.py * Update vision.py * Update _utils.py * Update vision.py * move forced float32 * Update _utils.py * Update _utils.py * Update _utils.py * Update _utils.py * move print * Update _utils.py * disable bfloat16 * Fix forced float32 * move float32 * Ensure trust_remote_code propegates down to unsloth_compile_transformers (#2075) * Update _utils.py * Show both `peft_error` and `autoconfig_error`, not just `autoconfig_error` (#2080) When loading a PEFT model fails, only the `autoconfig_error` is shown. Instead of the `peft_error`, which is what really matters when we're trying to load a PEFT adapter, the user will see something like this: ``` RuntimeError: Unrecognized model in my_model. Should have a `model_type` key in its config.json, or contain one of the following strings in its name: albert, align, altclip, ... ``` This PR just changes it so `autoconfig_error` and `peft_error` are both displayed. * fix error message (#2046) * Update vision.py * Update _utils.py * Update pyproject.toml * Update __init__.py * Update __init__.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update rl_replacements.py * Update vision.py * Update rl_replacements.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Remove double generate patch * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update mapper.py * Update vision.py * fix: config.torch_dtype in LlamaModel_fast_forward_inference (#2091) * fix: config.torch_dtype in LlamaModel_fast_forward_inference * Update llama.py * update for consistency --------- Co-authored-by: Daniel Han <danielhanchen@gmail.com> * versioning * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * model_type_arch * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py --------- Co-authored-by: Kareem <81531392+KareemMusleh@users.noreply.github.com> Co-authored-by: Wilson Wu <140025193+wiwu2390@users.noreply.github.com> Co-authored-by: Akshay Behl <126911424+Captain-T2004@users.noreply.github.com> Co-authored-by: Nino Risteski <95188570+NinoRisteski@users.noreply.github.com> Co-authored-by: Mukkesh Ganesh <mukmckenzie@gmail.com> Co-authored-by: Xander Hawthorne <167850078+CuppaXanax@users.noreply.github.com> Co-authored-by: Isaac Breen <isaac.breen@icloud.com> Co-authored-by: lurf21 <93976703+lurf21@users.noreply.github.com>
This commit is contained in:
Daniel Han
GitHub
parent
cac587d6fb
commit
81289dacca
6 changed files with 87 additions and 40 deletions
|
|
@ -37,7 +37,7 @@ triton = [
|
|||
]
|
||||
|
||||
huggingface = [
|
||||
"unsloth_zoo>=2025.3.13",
|
||||
"unsloth_zoo>=2025.3.14",
|
||||
"packaging",
|
||||
"tyro",
|
||||
"transformers>=4.46.1,!=4.47.0",
|
||||
|
|
@ -351,7 +351,7 @@ colab-ampere-torch220 = [
|
|||
"flash-attn>=2.6.3",
|
||||
]
|
||||
colab-new = [
|
||||
"unsloth_zoo>=2025.3.13",
|
||||
"unsloth_zoo>=2025.3.14",
|
||||
"packaging",
|
||||
"tyro",
|
||||
"transformers>=4.46.1,!=4.47.0",
|
||||
|
|
|
|||
|
|
@ -198,7 +198,7 @@ pass
|
|||
# Check for unsloth_zoo
|
||||
try:
|
||||
unsloth_zoo_version = importlib_version("unsloth_zoo")
|
||||
if Version(unsloth_zoo_version) < Version("2025.3.13"):
|
||||
if Version(unsloth_zoo_version) < Version("2025.3.14"):
|
||||
print(
|
||||
"Unsloth: Updating Unsloth-Zoo utilies to the latest version.\n"\
|
||||
"To disable this, set `os.environ['UNSLOTH_DISABLE_AUTO_UPDATES'] = '1'`"
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
__version__ = "2025.3.15"
|
||||
__version__ = "2025.3.16"
|
||||
|
||||
__all__ = [
|
||||
"SUPPORTS_BFLOAT16",
|
||||
|
|
@ -1177,6 +1177,7 @@ def unsloth_compile_transformers(
|
|||
return
|
||||
if disable: return
|
||||
|
||||
model_types = list(dict().fromkeys(model_types).keys())
|
||||
for model_type in model_types:
|
||||
_unsloth_compile_transformers(
|
||||
model_type,
|
||||
|
|
|
|||
|
|
@ -652,13 +652,7 @@ def LlamaModel_fast_forward(
|
|||
if inputs_embeds is None:
|
||||
inputs_embeds = self.embed_tokens(input_ids)
|
||||
|
||||
# inputs_embeds = inputs_embeds.to(self.config.torch_dtype)
|
||||
torch_dtype = __DTYPE_MAP.get(self.config.torch_dtype, None)
|
||||
if torch_dtype is not None:
|
||||
inputs_embeds = inputs_embeds.to(torch_dtype)
|
||||
else:
|
||||
raise TypeError("Unsloth: torch_dtype for models is not bfloat16, float16 or float32!")
|
||||
pass
|
||||
inputs_embeds = inputs_embeds.to(_get_dtype(self.config.torch_dtype))
|
||||
|
||||
# Normalized from Gemma
|
||||
IS_GEMMA = self.config.model_type.startswith("gemma")
|
||||
|
|
@ -924,7 +918,7 @@ def LlamaModel_fast_forward_inference(
|
|||
mlp_size = self.config.intermediate_size
|
||||
|
||||
X = self.model.embed_tokens(input_ids)
|
||||
X = X.to(self.config.torch_dtype)
|
||||
X = X.to(_get_dtype(self.config.torch_dtype))
|
||||
bsz, q_len, hd = X.shape
|
||||
assert(q_len == 1)
|
||||
# Get saved buffers to reduce memory movement
|
||||
|
|
@ -2457,12 +2451,6 @@ class FastLlamaModel:
|
|||
# Add for_inference and for_training
|
||||
model.for_training = functools.partial(FastLlamaModel.for_training, model)
|
||||
model.for_inference = functools.partial(FastLlamaModel.for_inference, model)
|
||||
|
||||
# Patch generate
|
||||
if model.generate.__name__ != "unsloth_fast_generate":
|
||||
model._old_generate = model.generate
|
||||
unsloth_fast_generate.__doc__ = model._old_generate.__doc__
|
||||
model.generate = types.MethodType(unsloth_fast_generate, model)
|
||||
return model
|
||||
pass
|
||||
|
||||
|
|
|
|||
|
|
@ -718,6 +718,16 @@ __INT_TO_FLOAT_MAPPER = \
|
|||
"allenai/OLMo-2-0325-32B-Instruct",
|
||||
"unsloth/OLMo-2-0325-32B-Instruct-bnb-4bit",
|
||||
),
|
||||
"unsloth/Mistral-Small-3.1-24B-Instruct-2503-unsloth-bnb-4bit" : (
|
||||
"unsloth/Mistral-Small-3.1-24B-Instruct-2503",
|
||||
"mistralai/Mistral-Small-3.1-24B-Instruct-2503",
|
||||
"unsloth/Mistral-Small-3.1-24B-Instruct-2503-bnb-4bit",
|
||||
),
|
||||
"unsloth/Mistral-Small-3.1-24B-Base-2503-unsloth-bnb-4bit" : (
|
||||
"unsloth/Mistral-Small-3.1-24B-Base-2503",
|
||||
"mistralai/Mistral-Small-3.1-24B-Base-2503",
|
||||
"unsloth/Mistral-Small-3.1-24B-Base-2503-bnb-4bit",
|
||||
),
|
||||
}
|
||||
|
||||
INT_TO_FLOAT_MAPPER = {}
|
||||
|
|
|
|||
|
|
@ -76,19 +76,34 @@ NUM_LOGITS_TO_KEEP = dict()
|
|||
global PROMPT_LOOPKUP
|
||||
PROMPT_LOOPKUP = dict()
|
||||
|
||||
from transformers import GenerationConfig, CompileConfig, HybridCache
|
||||
_compile_config = CompileConfig(
|
||||
fullgraph = False,
|
||||
dynamic = None,
|
||||
mode = "reduce-overhead",
|
||||
)
|
||||
_compile_config.disable = True # Must set manually
|
||||
|
||||
from unsloth_zoo.vllm_utils import (
|
||||
convert_lora_modules,
|
||||
return_lora_modules,
|
||||
)
|
||||
|
||||
def unsloth_base_fast_generate(
|
||||
self,
|
||||
*args,
|
||||
**kwargs,
|
||||
):
|
||||
if len(args) != 0:
|
||||
x = args[0]
|
||||
input_ids = args[0]
|
||||
elif "input_ids" in kwargs:
|
||||
x = kwargs["input_ids"]
|
||||
input_ids = kwargs["input_ids"]
|
||||
elif "input" in kwargs:
|
||||
input_ids = kwargs["input_ids"]
|
||||
else:
|
||||
raise TypeError("Unsloth: You need to pass in input_ids to .generate!")
|
||||
assert(type(x) is torch.Tensor)
|
||||
bsz = x.shape[0]
|
||||
assert(type(input_ids) is torch.Tensor)
|
||||
bsz = input_ids.shape[0]
|
||||
|
||||
FastBaseModel.for_inference(self)
|
||||
dtype = _get_dtype(self.config.torch_dtype)
|
||||
|
|
@ -101,8 +116,8 @@ def unsloth_base_fast_generate(
|
|||
is_vlm = is_vlm or hasattr(self.config, "vision_config")
|
||||
arch = self.config.architectures[0]
|
||||
|
||||
# Remove token_type_ids
|
||||
kwargs.pop("token_type_ids", None)
|
||||
# Remove token_type_ids - WRONG for Gemma 3 since bidirectional attention
|
||||
# kwargs.pop("token_type_ids", None)
|
||||
|
||||
# VLMs do not allow logits_to_keep
|
||||
global NUM_LOGITS_TO_KEEP
|
||||
|
|
@ -146,20 +161,58 @@ def unsloth_base_fast_generate(
|
|||
try: kwargs["pixel_values"] = kwargs["pixel_values"].to(dtype)
|
||||
except: pass
|
||||
|
||||
if "use_cache" not in kwargs: kwargs["use_cache"] = True
|
||||
|
||||
# Mixed precision autocast
|
||||
if os.environ.get("UNSLOTH_FORCE_FLOAT32", "0") == "1":
|
||||
autocaster = torch.autocast(device_type = "cuda", dtype = dtype)
|
||||
autocaster = torch.autocast(device_type = "cuda", dtype = torch.float16)
|
||||
dtype = torch.float16
|
||||
else:
|
||||
autocaster = torch.autocast(device_type = "cuda", dtype = dtype)
|
||||
with torch.inference_mode(), autocaster:
|
||||
try:
|
||||
|
||||
# Prepare LoRA
|
||||
# state_dict = convert_lora_modules(self, dtype = dtype)
|
||||
|
||||
# Set compile dynamic shapes
|
||||
torch._dynamo.mark_static(input_ids, 0)
|
||||
torch._dynamo.mark_dynamic(input_ids, 1)
|
||||
if "attention_mask" in kwargs:
|
||||
torch._dynamo.mark_static(kwargs["attention_mask"], 0)
|
||||
torch._dynamo.mark_dynamic(kwargs["attention_mask"], 1)
|
||||
if "token_type_ids" in kwargs:
|
||||
torch._dynamo.mark_static(kwargs["token_type_ids"], 0)
|
||||
torch._dynamo.mark_dynamic(kwargs["token_type_ids"], 1)
|
||||
|
||||
# Fix generation_config
|
||||
# Use hybrid if sliding window seen, otherwise try static
|
||||
cache_implementation = getattr(self.config, "cache_implementation", None)
|
||||
if getattr(self, "_supports_static_cache", True):
|
||||
cache_implementation = "static"
|
||||
else:
|
||||
cache_implementation = None
|
||||
if cache_implementation is not None:
|
||||
swa = getattr(getattr(self.config, "text_config", self.config), "sliding_window", None)
|
||||
if swa == 0 or type(swa) is not int:
|
||||
cache_implementation = "static"
|
||||
else:
|
||||
cache_implementation = "hybrid"
|
||||
if "generation_config" in kwargs:
|
||||
kwargs["generation_config"].cache_implementation = cache_implementation
|
||||
kwargs["generation_config"].compile_config = _compile_config
|
||||
else:
|
||||
kwargs["cache_implementation"] = cache_implementation
|
||||
kwargs["compile_config"] = _compile_config
|
||||
pass
|
||||
|
||||
try:
|
||||
with torch.inference_mode(), autocaster:
|
||||
output = self._old_generate(*args, **kwargs)
|
||||
except:
|
||||
PROMPT_LOOPKUP[arch] = False
|
||||
kwargs.pop("prompt_lookup_num_tokens", None)
|
||||
except:
|
||||
PROMPT_LOOPKUP[arch] = False
|
||||
kwargs.pop("prompt_lookup_num_tokens", None)
|
||||
with torch.inference_mode(), autocaster:
|
||||
output = self._old_generate(*args, **kwargs)
|
||||
finally:
|
||||
pass
|
||||
# return_lora_modules(self, state_dict, torch.float32)
|
||||
pass
|
||||
|
||||
FastBaseModel.for_training(self)
|
||||
|
|
@ -203,8 +256,9 @@ class FastBaseModel:
|
|||
except: vllm_version = ""
|
||||
|
||||
model_type_arch = model_types[0]
|
||||
if model_type_arch == "siglip" and len(model_types) != 1:
|
||||
model_type_arch = model_types[1]
|
||||
if model_type_arch == "siglip":
|
||||
for model_type_arch in model_types:
|
||||
if model_type_arch != "siglip": break
|
||||
|
||||
statistics = \
|
||||
f"==((====))== Unsloth {__version__}: Fast {model_type_arch.title()} patching. Transformers: {transformers_version}.{vllm_version}\n"\
|
||||
|
|
@ -543,12 +597,6 @@ class FastBaseModel:
|
|||
# Add for_inference and for_training
|
||||
model.for_training = functools.partial(FastBaseModel.for_training, model)
|
||||
model.for_inference = functools.partial(FastBaseModel.for_inference, model)
|
||||
|
||||
# Patch generate
|
||||
if model.generate.__name__ != "unsloth_base_fast_generate":
|
||||
model._old_generate = model.generate
|
||||
unsloth_base_fast_generate.__doc__ = model._old_generate.__doc__
|
||||
model.generate = types.MethodType(unsloth_base_fast_generate, model)
|
||||
return model
|
||||
pass
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue