Compare commits
3 commits
main
...
dh/recover
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ae38c3639d | ||
|
|
c823e52b2d | ||
|
|
9376c9064b |
5 changed files with 22 additions and 3 deletions
|
|
@ -1667,14 +1667,18 @@ def _change_system_message(template: str, type_chat_template: str, system_messag
|
|||
if has_placeholder:
|
||||
if system_message is None:
|
||||
raise ValueError("Unsloth: You need to provide a system message for custom templates.")
|
||||
new_template = re.sub(system_message_pattern, system_message, template)
|
||||
# Escape single quotes to prevent Jinja2 template syntax errors
|
||||
escaped_message = system_message.replace("'", "\\'")
|
||||
new_template = re.sub(system_message_pattern, escaped_message, template)
|
||||
return new_template, system_message
|
||||
|
||||
return template, system_message
|
||||
|
||||
# For predefined templates with default system message
|
||||
message_to_use = system_message if system_message is not None else default_system_message
|
||||
new_template = re.sub(system_message_pattern, message_to_use, template)
|
||||
# Escape single quotes to prevent Jinja2 template syntax errors
|
||||
escaped_message = message_to_use.replace("'", "\\'")
|
||||
new_template = re.sub(system_message_pattern, escaped_message, template)
|
||||
|
||||
return new_template, message_to_use
|
||||
|
||||
|
|
|
|||
|
|
@ -2331,6 +2331,7 @@ class FastLlamaModel:
|
|||
model_name,
|
||||
token = token,
|
||||
attn_implementation = "sdpa",
|
||||
revision = revision,
|
||||
)
|
||||
model_config.model_name = model_name
|
||||
model_max_seq_length = model_config.max_position_embeddings
|
||||
|
|
@ -2424,6 +2425,7 @@ class FastLlamaModel:
|
|||
max_position_embeddings = max_position_embeddings,
|
||||
trust_remote_code = trust_remote_code,
|
||||
attn_implementation = preferred_attn_impl,
|
||||
revision = revision,
|
||||
**kwargs,
|
||||
)
|
||||
elif not fast_inference:
|
||||
|
|
@ -2436,6 +2438,7 @@ class FastLlamaModel:
|
|||
max_position_embeddings = max_position_embeddings,
|
||||
trust_remote_code = trust_remote_code,
|
||||
attn_implementation = preferred_attn_impl,
|
||||
revision = revision,
|
||||
**kwargs,
|
||||
)
|
||||
model.fast_generate = make_fast_generate_wrapper(model.generate)
|
||||
|
|
@ -2468,6 +2471,7 @@ class FastLlamaModel:
|
|||
disable_log_stats = disable_log_stats,
|
||||
use_bitsandbytes = load_in_4bit,
|
||||
unsloth_vllm_standby = unsloth_vllm_standby,
|
||||
revision = revision,
|
||||
fp8_mode = fp8_mode,
|
||||
)
|
||||
for allowed_arg in allowed_args:
|
||||
|
|
@ -2505,6 +2509,7 @@ class FastLlamaModel:
|
|||
token = token,
|
||||
trust_remote_code = trust_remote_code,
|
||||
fix_tokenizer = fix_tokenizer,
|
||||
revision = revision,
|
||||
)
|
||||
|
||||
model, tokenizer = patch_tokenizer(model, tokenizer)
|
||||
|
|
|
|||
|
|
@ -553,6 +553,7 @@ class FastLanguageModel(FastLlamaModel):
|
|||
model_config = AutoConfig.from_pretrained(
|
||||
model_name,
|
||||
token = token,
|
||||
revision = revision,
|
||||
trust_remote_code = trust_remote_code,
|
||||
)
|
||||
|
||||
|
|
@ -1304,6 +1305,7 @@ class FastModel(FastBaseModel):
|
|||
model_config = AutoConfig.from_pretrained(
|
||||
model_name,
|
||||
token = token,
|
||||
revision = revision,
|
||||
trust_remote_code = trust_remote_code,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -421,6 +421,7 @@ class FastBaseModel:
|
|||
auto_config = None,
|
||||
offload_embedding = False,
|
||||
float32_mixed_precision = None, # Forces float32 mixed precision
|
||||
revision = None,
|
||||
# vLLM parameters
|
||||
fast_inference = False,
|
||||
gpu_memory_utilization = 0.5,
|
||||
|
|
@ -720,6 +721,7 @@ class FastBaseModel:
|
|||
model_name,
|
||||
token = token,
|
||||
trust_remote_code = trust_remote_code,
|
||||
revision = revision,
|
||||
)
|
||||
if hasattr(auto_config, "quantization_config"):
|
||||
from transformers.quantizers.auto import (
|
||||
|
|
@ -776,12 +778,12 @@ class FastBaseModel:
|
|||
model_name,
|
||||
token = token,
|
||||
trust_remote_code = trust_remote_code,
|
||||
revision = revision,
|
||||
)
|
||||
setattr(auto_config, "_attn_implementation", config_attn_impl)
|
||||
if hasattr(auto_config, "attn_implementation"):
|
||||
setattr(auto_config, "attn_implementation", config_attn_impl)
|
||||
model_config = auto_config
|
||||
|
||||
verify_fp8_support_if_applicable(model_config)
|
||||
|
||||
raise_handler = RaiseUninitialized()
|
||||
|
|
@ -796,6 +798,7 @@ class FastBaseModel:
|
|||
# quantization_config = bnb_config,
|
||||
token = token,
|
||||
trust_remote_code = trust_remote_code,
|
||||
revision = revision,
|
||||
# attn_implementation = attn_implementation,
|
||||
**kwargs,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -503,6 +503,7 @@ def _load_correct_tokenizer(
|
|||
trust_remote_code = False,
|
||||
cache_dir = "huggingface_tokenizers_cache",
|
||||
fix_tokenizer = True,
|
||||
revision = None,
|
||||
):
|
||||
if IS_COLAB_ENVIRONMENT:
|
||||
cache_dir = cache_dir
|
||||
|
|
@ -528,6 +529,7 @@ def _load_correct_tokenizer(
|
|||
legacy = False,
|
||||
from_slow = True,
|
||||
cache_dir = cache_dir,
|
||||
revision = revision,
|
||||
)
|
||||
except:
|
||||
slow_tokenizer = None
|
||||
|
|
@ -546,6 +548,7 @@ def _load_correct_tokenizer(
|
|||
token = token,
|
||||
trust_remote_code = trust_remote_code,
|
||||
cache_dir = cache_dir,
|
||||
revision = revision,
|
||||
)
|
||||
|
||||
if not fix_tokenizer or tokenizer_name in IGNORED_TOKENIZER_NAMES:
|
||||
|
|
@ -587,6 +590,7 @@ def load_correct_tokenizer(
|
|||
trust_remote_code = False,
|
||||
cache_dir = "huggingface_tokenizers_cache",
|
||||
fix_tokenizer = True,
|
||||
revision = None,
|
||||
):
|
||||
tokenizer = _load_correct_tokenizer(
|
||||
tokenizer_name = tokenizer_name,
|
||||
|
|
@ -596,6 +600,7 @@ def load_correct_tokenizer(
|
|||
trust_remote_code = trust_remote_code,
|
||||
cache_dir = cache_dir,
|
||||
fix_tokenizer = fix_tokenizer,
|
||||
revision = revision,
|
||||
)
|
||||
|
||||
### 1. Fixup tokenizer's chat_template
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue