Fix Whisper, ModernBERT (#2565)
* Update vision.py * Update vision.py * Update mapper.py * Update vision.py * fix: config.torch_dtype in LlamaModel_fast_forward_inference (#2091) * fix: config.torch_dtype in LlamaModel_fast_forward_inference * Update llama.py * update for consistency --------- Co-authored-by: Daniel Han <danielhanchen@gmail.com> * versioning * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * model_type_arch * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update loader.py * check * Update _utils.py * Update loader.py * Update loader.py * Remove prints * Update README.md typo * Update _utils.py * Update _utils.py * versioning * Update _utils.py * Update _utils.py * Update _utils.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update vision.py * HF Transfer * fix(utils): add missing importlib import to fix NameError (#2134) This commit fixes a NameError that occurs when `importlib` is referenced in _utils.py without being imported, especially when UNSLOTH_USE_MODELSCOPE=1 is enabled. By adding the missing import statement, the code will no longer throw a NameError. * Add QLoRA Train and Merge16bit Test (#2130) * add reference and unsloth lora merging tests * add test / dataset printing to test scripts * allow running tests from repo root * add qlora test readme * more readme edits * ruff formatting * additional readme comments * forgot to add actual tests * add apache license * Update pyproject.toml * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update loader.py * Update loader.py * Revert * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Bug fix * Update mapper.py * check SDPA for Mistral 3, Pixtral * Update vision.py * Versioning * Update rl_replacements.py * Update README.md * add model registry * move hf hub utils to unsloth/utils * refactor global model info dicts to dataclasses * fix dataclass init * fix llama registration * remove deprecated key function * start registry reog * add llama vision * quant types -> Enum * remap literal quant types to QuantType Enum * add llama model registration * fix quant tag mapping * add qwen2.5 models to registry * add option to include original model in registry * handle quant types per model size * separate registration of base and instruct llama3.2 * add QwenQVQ to registry * add gemma3 to registry * add phi * add deepseek v3 * add deepseek r1 base * add deepseek r1 zero * add deepseek distill llama * add deepseek distill models * remove redundant code when constructing model names * add mistral small to registry * rename model registration methods * rename deepseek registration methods * refactor naming for mistral and phi * add global register models * refactor model registration tests for new registry apis * add model search method * remove deprecated registration api * add quant type test * add registry readme * make llama registration more specific * clear registry when executing individual model registration file * more registry readme updates * Update _auto_install.py * Llama4 * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Synthetic data * Update mapper.py * Xet and Synthetic * Update synthetic.py * Update loader.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update pyproject.toml * Delete .gitignore * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update _utils.py * Update pyproject.toml * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update chat_templates.py * Seasame force float16 / float32 * Fix Seasame * Update loader.py * Update vision.py * Update vision.py * Update vision.py * Update loader.py * is_multimodal * Update loader.py * Update loader.py * Update loader.py * Update loader.py * Update vision.py * Update vision.py * Update vision.py * UNSLOTH_DISABLE_STATIC_GENERATION * Update vision.py * Auto vision detection * Sesame * Whisper * Update loader.py * Update loader.py * Update loader.py * Update mapper.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update loader.py * Update loader.py * Update loader.py * Update loader.py * Update _utils.py --------- Co-authored-by: lurf21 <93976703+lurf21@users.noreply.github.com> Co-authored-by: Jack Shi Wei Lun <87535974+jackswl@users.noreply.github.com> Co-authored-by: naliazheli <nalia0316@gmail.com> Co-authored-by: jeromeku <jerome.ku@gmail.com> Co-authored-by: Michael Han <107991372+shimmyshimmer@users.noreply.github.com>
This commit is contained in:
Daniel Han
GitHub
parent
2a1caa746b
commit
4d5f2172f4
4 changed files with 63 additions and 32 deletions
|
|
@ -12,7 +12,7 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
__version__ = "2025.5.5"
|
||||
__version__ = "2025.5.6"
|
||||
|
||||
__all__ = [
|
||||
"SUPPORTS_BFLOAT16",
|
||||
|
|
|
|||
|
|
@ -461,6 +461,12 @@ except:
|
|||
from transformers import AutoModelForVision2Seq
|
||||
pass
|
||||
|
||||
DISABLE_COMPILE_MODEL_NAMES = [
|
||||
"aya-vision",
|
||||
"modernbert",
|
||||
"granite-vision",
|
||||
]
|
||||
|
||||
|
||||
class FastModel(FastBaseModel):
|
||||
@staticmethod
|
||||
|
|
@ -521,33 +527,35 @@ class FastModel(FastBaseModel):
|
|||
model_name = get_model_name(model_name, load_in_4bit)
|
||||
|
||||
# Check versions
|
||||
lowered_model_name = model_name.lower()
|
||||
LATEST = '\nPlease use transformers via `pip install --no-deps git+https://github.com/huggingface/transformers.git`'
|
||||
NIGHTLY = '\nPlease use nightly transformers via pip install --upgrade "transformers>=4.49.0"`'
|
||||
if "pixtral" in model_name.lower() and transformers_version < Version("4.49.0"):
|
||||
if "pixtral" in lowered_model_name and transformers_version < Version("4.49.0"):
|
||||
raise RuntimeError("Unsloth: Pixtral only works on transformers >= 4.49.0." + LATEST)
|
||||
elif "qwen2.5" in model_name.lower() and transformers_version < Version("4.49.0"):
|
||||
elif "qwen2.5" in lowered_model_name and transformers_version < Version("4.49.0"):
|
||||
raise RuntimeError("Unsloth: Qwen 2.5 only works on transformers >= 4.49.0." + LATEST)
|
||||
elif "aya-vision" in model_name.lower():
|
||||
# Disable compiling for now - errors out!
|
||||
os.environ["UNSLOTH_COMPILE_DISABLE"] = "1"
|
||||
if transformers_version < Version("4.50.0.dev0"):
|
||||
raise RuntimeError("Unsloth: Aya Vision only works on transformers >= 4.50.0." + NIGHTLY)
|
||||
elif "gemma-3" in model_name.lower() and transformers_version < Version("4.50.0.dev0"):
|
||||
elif "gemma-3" in lowered_model_name and transformers_version < Version("4.50.0.dev0"):
|
||||
raise RuntimeError("Unsloth: Gemma 3 only works on transformers >= 4.50.0." + NIGHTLY)
|
||||
elif "c4ai-command-a-03-2025" in model_name.lower() and transformers_version < Version("4.50.0.dev0"):
|
||||
elif "c4ai-command-a-03-2025" in lowered_model_name and transformers_version < Version("4.50.0.dev0"):
|
||||
raise RuntimeError("Unsloth: Cohere's Command model only works on transformers >= 4.50.0." + NIGHTLY)
|
||||
elif "granite-vision" in model_name.lower():
|
||||
# Disable compiling for now - errors out!
|
||||
os.environ["UNSLOTH_COMPILE_DISABLE"] = "1"
|
||||
if transformers_version < Version("4.50.0.dev0"):
|
||||
raise RuntimeError("Unsloth: Granite Vision only works on transformers >= 4.50.0." + NIGHTLY)
|
||||
elif "csm-1b" in model_name.lower():
|
||||
elif "csm-1b" in lowered_model_name:
|
||||
os.environ["UNSLOTH_DISABLE_STATIC_GENERATION"] = "1" # Sesame fails
|
||||
os.environ["UNSLOTH_FORCE_CUSTOM_DTYPE"] = "torch.float16;if name.endswith(('_proj', 'fc1', 'fc2', 'codebook', 'head')): module.to(torch.float16)"
|
||||
elif "olmo-2" in model_name.lower() and transformers_version < Version("4.50.0.dev0"):
|
||||
elif "olmo-2" in lowered_model_name and transformers_version < Version("4.50.0.dev0"):
|
||||
raise RuntimeError("Unsloth: OLMo-2 only works on transformers >= 4.50.0." + NIGHTLY)
|
||||
elif "whisper" in model_name.lower():
|
||||
os.environ["UNSLOTH_DISABLE_STATIC_GENERATION"] = "1" # Whisper fails
|
||||
else:
|
||||
for check_model_name in DISABLE_COMPILE_MODEL_NAMES:
|
||||
if check_model_name in lowered_model_name:
|
||||
os.environ["UNSLOTH_COMPILE_DISABLE"] = "1"
|
||||
os.environ["UNSLOTH_DISABLE_STATIC_GENERATION"] = "1"
|
||||
if transformers_version < Version("4.50.0.dev0"):
|
||||
raise RuntimeError(f"Unsloth: {check_model_name} only works on transformers >= 4.50.0." + NIGHTLY)
|
||||
break
|
||||
pass
|
||||
|
||||
if auto_model is not None:
|
||||
# All other models need to disable static cache
|
||||
os.environ["UNSLOTH_DISABLE_STATIC_GENERATION"] = "1"
|
||||
pass
|
||||
|
||||
if USE_MODELSCOPE and not os.path.exists(model_name):
|
||||
|
|
|
|||
|
|
@ -728,16 +728,6 @@ __INT_TO_FLOAT_MAPPER = \
|
|||
"mistralai/Mistral-Small-3.1-24B-Base-2503",
|
||||
"unsloth/Mistral-Small-3.1-24B-Base-2503-bnb-4bit",
|
||||
),
|
||||
"unsloth/orpheus-3b-0.1-pretrained-unsloth-bnb-4bit" : (
|
||||
"unsloth/orpheus-3b-0.1-pretrained",
|
||||
"canopylabs/orpheus-3b-0.1-pretrained",
|
||||
"unsloth/orpheus-3b-0.1-pretrained-bnb-4bit",
|
||||
),
|
||||
"unsloth/orpheus-3b-0.1-ft-unsloth-bnb-4bit" : (
|
||||
"unsloth/orpheus-3b-0.1-ft",
|
||||
"canopylabs/orpheus-3b-0.1-ft",
|
||||
"unsloth/orpheus-3b-0.1-ft-bnb-4bit",
|
||||
),
|
||||
"unsloth/Qwen3-0.6B-unsloth-bnb-4bit" : (
|
||||
"unsloth/Qwen3-0.6B",
|
||||
"Qwen/Qwen3-0.6B",
|
||||
|
|
@ -817,6 +807,16 @@ __INT_TO_FLOAT_MAPPER = \
|
|||
"microsoft/Phi-4-mini-reasoning",
|
||||
"unsloth/phi-4-mini-reasoning-bnb-4bit",
|
||||
),
|
||||
"unsloth/orpheus-3b-0.1-pretrained-unsloth-bnb-4bit" : (
|
||||
"unsloth/orpheus-3b-0.1-pretrained",
|
||||
"canopylabs/orpheus-3b-0.1-pretrained",
|
||||
"unsloth/orpheus-3b-0.1-pretrained-bnb-4bit",
|
||||
),
|
||||
"unsloth/orpheus-3b-0.1-ft-unsloth-bnb-4bit" : (
|
||||
"unsloth/orpheus-3b-0.1-ft",
|
||||
"canopylabs/orpheus-3b-0.1-ft",
|
||||
"unsloth/orpheus-3b-0.1-ft-bnb-4bit",
|
||||
),
|
||||
"unsloth/csm-1b" : (
|
||||
"unsloth/csm-1b",
|
||||
"sesame/csm-1b",
|
||||
|
|
@ -837,6 +837,18 @@ __INT_TO_FLOAT_MAPPER = \
|
|||
"unsloth/CrisperWhisper",
|
||||
"nyrahealth/CrisperWhisper",
|
||||
),
|
||||
"unsloth/Llasa-1B" : (
|
||||
"unsloth/Llasa-1B",
|
||||
"HKUSTAudio/Llasa-1B",
|
||||
),
|
||||
"unsloth/Spark-TTS-0.5B" : (
|
||||
"unsloth/Spark-TTS-0.5B",
|
||||
"SparkAudio/Spark-TTS-0.5B",
|
||||
),
|
||||
"unsloth/Llama-OuteTTS-1.0-1B" : (
|
||||
"unsloth/Llama-OuteTTS-1.0-1B",
|
||||
"OuteAI/Llama-OuteTTS-1.0-1B",
|
||||
),
|
||||
}
|
||||
|
||||
INT_TO_FLOAT_MAPPER = {}
|
||||
|
|
|
|||
|
|
@ -95,8 +95,18 @@ def unsloth_base_fast_generate(
|
|||
input_ids = kwargs["input_ids"]
|
||||
elif "input" in kwargs:
|
||||
input_ids = kwargs["input_ids"]
|
||||
elif "input_features" in kwargs:
|
||||
input_ids = kwargs["input_features"]
|
||||
elif "input_embeds" in kwargs:
|
||||
input_ids = kwargs["input_embeds"]
|
||||
elif "inputs" in kwargs:
|
||||
input_ids = kwargs["inputs"]
|
||||
else:
|
||||
raise TypeError("Unsloth: You need to pass in input_ids to .generate!")
|
||||
key = next(iter(kwargs.keys()))
|
||||
if type(kwargs["key"]) is not torch.Tensor:
|
||||
raise TypeError("Unsloth: You need to pass in input_ids to .generate!")
|
||||
input_ids = kwargs[key]
|
||||
pass
|
||||
assert(type(input_ids) is torch.Tensor)
|
||||
bsz = input_ids.shape[0]
|
||||
|
||||
|
|
@ -203,10 +213,11 @@ def unsloth_base_fast_generate(
|
|||
|
||||
if "generation_config" in kwargs:
|
||||
kwargs["generation_config"].cache_implementation = cache_implementation
|
||||
kwargs["generation_config"].compile_config = _compile_config if cache_implementation is not None else None
|
||||
if cache_implementation is not None:
|
||||
kwargs["generation_config"].compile_config = _compile_config
|
||||
else:
|
||||
kwargs["cache_implementation"] = cache_implementation
|
||||
if cache_implementation:
|
||||
if cache_implementation is not None:
|
||||
kwargs["compile_config"] = _compile_config
|
||||
pass
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue