TTS (#2545)
* Update rl_replacements.py * Update vision.py * Update rl_replacements.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Remove double generate patch * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update mapper.py * Update vision.py * fix: config.torch_dtype in LlamaModel_fast_forward_inference (#2091) * fix: config.torch_dtype in LlamaModel_fast_forward_inference * Update llama.py * update for consistency --------- Co-authored-by: Daniel Han <danielhanchen@gmail.com> * versioning * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * model_type_arch * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update loader.py * check * Update _utils.py * Update loader.py * Update loader.py * Remove prints * Update README.md typo * Update _utils.py * Update _utils.py * versioning * Update _utils.py * Update _utils.py * Update _utils.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update vision.py * HF Transfer * fix(utils): add missing importlib import to fix NameError (#2134) This commit fixes a NameError that occurs when `importlib` is referenced in _utils.py without being imported, especially when UNSLOTH_USE_MODELSCOPE=1 is enabled. By adding the missing import statement, the code will no longer throw a NameError. * Add QLoRA Train and Merge16bit Test (#2130) * add reference and unsloth lora merging tests * add test / dataset printing to test scripts * allow running tests from repo root * add qlora test readme * more readme edits * ruff formatting * additional readme comments * forgot to add actual tests * add apache license * Update pyproject.toml * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update loader.py * Update loader.py * Revert * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Bug fix * Update mapper.py * check SDPA for Mistral 3, Pixtral * Update vision.py * Versioning * Update rl_replacements.py * Update README.md * add model registry * move hf hub utils to unsloth/utils * refactor global model info dicts to dataclasses * fix dataclass init * fix llama registration * remove deprecated key function * start registry reog * add llama vision * quant types -> Enum * remap literal quant types to QuantType Enum * add llama model registration * fix quant tag mapping * add qwen2.5 models to registry * add option to include original model in registry * handle quant types per model size * separate registration of base and instruct llama3.2 * add QwenQVQ to registry * add gemma3 to registry * add phi * add deepseek v3 * add deepseek r1 base * add deepseek r1 zero * add deepseek distill llama * add deepseek distill models * remove redundant code when constructing model names * add mistral small to registry * rename model registration methods * rename deepseek registration methods * refactor naming for mistral and phi * add global register models * refactor model registration tests for new registry apis * add model search method * remove deprecated registration api * add quant type test * add registry readme * make llama registration more specific * clear registry when executing individual model registration file * more registry readme updates * Update _auto_install.py * Llama4 * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Synthetic data * Update mapper.py * Xet and Synthetic * Update synthetic.py * Update loader.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update pyproject.toml * Delete .gitignore * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update _utils.py * Update pyproject.toml * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update chat_templates.py * Seasame force float16 / float32 * Fix Seasame * Update loader.py * Update vision.py * Update vision.py * Update vision.py * Update loader.py * is_multimodal * Update loader.py * Update loader.py * Update loader.py * Update loader.py * Update vision.py * Update vision.py * Update vision.py * UNSLOTH_DISABLE_STATIC_GENERATION * Update vision.py * Auto vision detection * Sesame * Whisper * Update loader.py * Update loader.py * Update loader.py --------- Co-authored-by: lurf21 <93976703+lurf21@users.noreply.github.com> Co-authored-by: Jack Shi Wei Lun <87535974+jackswl@users.noreply.github.com> Co-authored-by: naliazheli <nalia0316@gmail.com> Co-authored-by: jeromeku <jerome.ku@gmail.com> Co-authored-by: Michael Han <107991372+shimmyshimmer@users.noreply.github.com>
This commit is contained in:
Daniel Han
GitHub
parent
f30b16b0d4
commit
532a43f828
6 changed files with 78 additions and 16 deletions
|
|
@ -37,7 +37,7 @@ triton = [
|
|||
]
|
||||
|
||||
huggingface = [
|
||||
"unsloth_zoo>=2025.5.5",
|
||||
"unsloth_zoo>=2025.5.6",
|
||||
"packaging",
|
||||
"tyro",
|
||||
"transformers==4.51.3,!=4.47.0",
|
||||
|
|
@ -381,7 +381,7 @@ colab-ampere-torch220 = [
|
|||
"flash-attn>=2.6.3",
|
||||
]
|
||||
colab-new = [
|
||||
"unsloth_zoo>=2025.5.5",
|
||||
"unsloth_zoo>=2025.5.6",
|
||||
"packaging",
|
||||
"tyro",
|
||||
"transformers==4.51.3,!=4.47.0",
|
||||
|
|
|
|||
|
|
@ -1036,9 +1036,21 @@ qwen3_template = \
|
|||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
|
||||
{%- for message in messages[::-1] %}
|
||||
{%- for forward_message in messages %}
|
||||
{%- set index = (messages|length - 1) - loop.index0 %}
|
||||
{%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
|
||||
{%- set message = messages[index] %}
|
||||
{%- set current_content = message.content if message.content is not none else '' %}
|
||||
{%- set tool_start = '<tool_response>' %}
|
||||
{%- set tool_start_length = tool_start|length %}
|
||||
{%- set start_of_message = current_content[:tool_start_length] %}
|
||||
{%- set tool_end = '</tool_response>' %}
|
||||
{%- set tool_end_length = tool_end|length %}
|
||||
{%- set start_pos = (current_content|length) - tool_end_length %}
|
||||
{%- if start_pos < 0 %}
|
||||
{%- set start_pos = 0 %}
|
||||
{%- endif %}
|
||||
{%- set end_of_message = current_content[start_pos:] %}
|
||||
{%- if ns.multi_step_tool and message.role == "user" and not(start_of_message == tool_start and end_of_message == tool_end) %}
|
||||
{%- set ns.multi_step_tool = false %}
|
||||
{%- set ns.last_query_index = index %}
|
||||
{%- endif %}
|
||||
|
|
@ -1053,8 +1065,9 @@ qwen3_template = \
|
|||
{%- set reasoning_content = message.reasoning_content %}
|
||||
{%- else %}
|
||||
{%- if '</think>' in message.content %}
|
||||
{%- set content = message.content.split('</think>')[-1].lstrip('\n') %}
|
||||
{%- set reasoning_content = message.content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
|
||||
{%- set content = (message.content.split('</think>')|last).lstrip('\n') %}
|
||||
{%- set reasoning_content = (message.content.split('</think>')|first).rstrip('\n') %}
|
||||
{%- set reasoning_content = (reasoning_content.split('<think>')|last).lstrip('\n') %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- if loop.index0 > ns.last_query_index %}
|
||||
|
|
@ -1110,7 +1123,7 @@ qwen3_template = \
|
|||
qwen3_ollama = \
|
||||
'''
|
||||
FROM {__FILE_LOCATION__}
|
||||
TEMPLATE """{{ if .Messages }}
|
||||
TEMPLATE """{{- if .Messages }}
|
||||
{{- if or .System .Tools }}<|im_start|>system
|
||||
{{- if .System }}
|
||||
{{ .System }}
|
||||
|
|
@ -1161,8 +1174,12 @@ For each function call, return a json object with function name and arguments wi
|
|||
{{ end }}<|im_start|>assistant
|
||||
{{ end }}{{ .Response }}{{ if .Response }}<|im_end|>{{ end }}"""
|
||||
PARAMETER stop "<|im_end|>"
|
||||
PARAMETER temperature 1.5
|
||||
PARAMETER min_p 0.1
|
||||
PARAMETER stop "<|im_start|>"
|
||||
PARAMETER temperature 0.6
|
||||
PARAMETER min_p 0.0
|
||||
PARAMETER top_k 20
|
||||
PARAMETER top_p 0.95
|
||||
PARAMETER repeat_penalty 1
|
||||
'''
|
||||
|
||||
qwen3_template_eos_token = "<|im_end|>"
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
__version__ = "2025.5.3"
|
||||
__version__ = "2025.5.4"
|
||||
|
||||
__all__ = [
|
||||
"SUPPORTS_BFLOAT16",
|
||||
|
|
|
|||
|
|
@ -541,10 +541,12 @@ class FastModel(FastBaseModel):
|
|||
if transformers_version < Version("4.50.0.dev0"):
|
||||
raise RuntimeError("Unsloth: Granite Vision only works on transformers >= 4.50.0." + NIGHTLY)
|
||||
elif "csm-1b" in model_name.lower():
|
||||
os.environ["UNSLOTH_COMPILE_DISABLE"] = "1"
|
||||
os.environ["UNSLOTH_DISABLE_FAST_GENERATION"] = "1"
|
||||
os.environ["UNSLOTH_DISABLE_STATIC_GENERATION"] = "1" # Sesame fails
|
||||
os.environ["UNSLOTH_FORCE_CUSTOM_DTYPE"] = "torch.float16;if name.endswith(('_proj', 'fc1', 'fc2', 'codebook', 'head')): module.to(torch.float16)"
|
||||
elif "olmo-2" in model_name.lower() and transformers_version < Version("4.50.0.dev0"):
|
||||
raise RuntimeError("Unsloth: OLMo-2 only works on transformers >= 4.50.0." + NIGHTLY)
|
||||
elif "whisper" in model_name.lower():
|
||||
os.environ["UNSLOTH_DISABLE_STATIC_GENERATION"] = "1" # Whisper fails
|
||||
pass
|
||||
|
||||
if USE_MODELSCOPE and not os.path.exists(model_name):
|
||||
|
|
|
|||
|
|
@ -817,6 +817,26 @@ __INT_TO_FLOAT_MAPPER = \
|
|||
"microsoft/Phi-4-mini-reasoning",
|
||||
"unsloth/phi-4-mini-reasoning-bnb-4bit",
|
||||
),
|
||||
"unsloth/csm-1b" : (
|
||||
"unsloth/csm-1b",
|
||||
"sesame/csm-1b",
|
||||
),
|
||||
"unsloth/whisper-large-v3" : (
|
||||
"unsloth/whisper-large-v3",
|
||||
"openai/whisper-large-v3",
|
||||
),
|
||||
"unsloth/whisper-large-v3-turbo" : (
|
||||
"unsloth/whisper-large-v3-turbo",
|
||||
"openai/whisper-large-v3-turbo",
|
||||
),
|
||||
"unsloth/whisper-small" : (
|
||||
"unsloth/whisper-small",
|
||||
"openai/whisper-small",
|
||||
),
|
||||
"unsloth/CrisperWhisper" : (
|
||||
"unsloth/CrisperWhisper",
|
||||
"nyrahealth/CrisperWhisper",
|
||||
),
|
||||
}
|
||||
|
||||
INT_TO_FLOAT_MAPPER = {}
|
||||
|
|
|
|||
|
|
@ -188,7 +188,10 @@ def unsloth_base_fast_generate(
|
|||
# Use hybrid if sliding window seen, otherwise try static
|
||||
cache_implementation = getattr(self.config, "cache_implementation", None)
|
||||
if getattr(self, "_supports_static_cache", True):
|
||||
cache_implementation = "static"
|
||||
if os.environ.get("UNSLOTH_DISABLE_STATIC_GENERATION", "0") == "0":
|
||||
cache_implementation = "static"
|
||||
else:
|
||||
cache_implementation = None
|
||||
else:
|
||||
cache_implementation = None
|
||||
if cache_implementation is not None:
|
||||
|
|
@ -199,10 +202,10 @@ def unsloth_base_fast_generate(
|
|||
cache_implementation = "hybrid"
|
||||
if "generation_config" in kwargs:
|
||||
kwargs["generation_config"].cache_implementation = cache_implementation
|
||||
kwargs["generation_config"].compile_config = _compile_config
|
||||
kwargs["generation_config"].compile_config = _compile_config if cache_implementation is not None else None
|
||||
else:
|
||||
kwargs["cache_implementation"] = cache_implementation
|
||||
kwargs["compile_config"] = _compile_config
|
||||
kwargs["compile_config"] = _compile_config if cache_implementation is not None else None
|
||||
pass
|
||||
|
||||
try:
|
||||
|
|
@ -310,6 +313,19 @@ class FastBaseModel:
|
|||
bnb_compute_dtype = torch.float16
|
||||
do_forced_float32 = True
|
||||
pass
|
||||
|
||||
# Check for custom data-types
|
||||
custom_datatype = None
|
||||
correct_dtype = None
|
||||
if os.environ.get("UNSLOTH_FORCE_CUSTOM_DTYPE", "") != "":
|
||||
custom_datatype = os.environ["UNSLOTH_FORCE_CUSTOM_DTYPE"]
|
||||
assert custom_datatype.count(";") == 1
|
||||
bnb_compute_dtype, custom_datatype = custom_datatype.split(";", 1)
|
||||
dtype = torch.float32
|
||||
bnb_compute_dtype = eval(bnb_compute_dtype)
|
||||
correct_dtype = bnb_compute_dtype
|
||||
pass
|
||||
|
||||
# Stop SDPA for some archs like Pixtral / Mistral3
|
||||
if not ("attn_implementation" in kwargs):
|
||||
kwargs["attn_implementation"] = "sdpa"
|
||||
|
|
@ -374,12 +390,18 @@ class FastBaseModel:
|
|||
# Return old flag
|
||||
os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = old_hf_transfer
|
||||
|
||||
# Edit data-types
|
||||
if custom_datatype is not None:
|
||||
for name, module in model.named_modules():
|
||||
exec(custom_datatype)
|
||||
pass
|
||||
|
||||
# Counteract saved tokenizers
|
||||
tokenizer_name = model_name if tokenizer_name is None else tokenizer_name
|
||||
is_vlm = (auto_model is AutoModelForVision2Seq)
|
||||
is_whisper = (whisper_language is not None and whisper_task is not None)
|
||||
auto_processor = AutoProcessor if (is_vlm or is_whisper) else AutoTokenizer
|
||||
if whisper_language and whisper_task:
|
||||
if (whisper_language and whisper_task) or auto_model.__name__.endswith("ForConditionalGeneration"):
|
||||
tokenizer = auto_processor.from_pretrained(
|
||||
tokenizer_name,
|
||||
padding_side = "right",
|
||||
|
|
@ -415,6 +437,7 @@ class FastBaseModel:
|
|||
downcast_rope = False,
|
||||
fix_embeddings = False,
|
||||
do_forced_float32 = do_forced_float32,
|
||||
correct_dtype = correct_dtype,
|
||||
)
|
||||
model, tokenizer = patch_tokenizer(model, tokenizer)
|
||||
model = post_patch_loss_function(model)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue