ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

check_tokenizer

Browse source
This commit is contained in:
Daniel Han-Chen 2024-01-09 23:40:43 +11:00
commit b52278199b
3 changed files with 24 additions and 16 deletions
Download patch file Download diff file Expand all files Collapse all files

19
unsloth/models/llama.py
View file

@ -619,6 +619,7 @@ class FastLlamaModel:
token = None,
device_map = "sequential",
rope_scaling = None,
check_tokenizer = True,
):
SUPPORTS_BFLOAT16 = torch.cuda.is_bf16_supported()
gpu_stats = torch.cuda.get_device_properties(0)
@ -703,14 +704,16 @@ class FastLlamaModel:
internal_model.max_seq_length = max_position_embeddings
# We check the tokenizer first for errors
tokenizer = check_tokenizer(
model = model,
tokenizer = tokenizer,
model_name = model_name,
model_max_length = max_seq_length,
padding_side = "right",
token = token,
)
if check_tokenizer:
tokenizer = check_tokenizer(
model = model,
tokenizer = tokenizer,
model_name = model_name,
model_max_length = max_seq_length,
padding_side = "right",
token = token,
)
pass
return model, tokenizer
pass

2
unsloth/models/loader.py
View file

@ -44,6 +44,7 @@ class FastLanguageModel(FastLlamaModel):
token = None,
device_map = "sequential",
rope_scaling = None,
check_tokenizer = True,
*args, **kwargs,
):
if not SUPPORTS_FOURBIT and model_name in FOURBIT_MAPPER:
@ -83,6 +84,7 @@ class FastLanguageModel(FastLlamaModel):
token = token,
device_map = device_map,
rope_scaling = rope_scaling,
check_tokenizer = check_tokenizer,
*args, **kwargs,
)
pass

19
unsloth/models/mistral.py
View file

@ -263,6 +263,7 @@ class FastMistralModel(FastLlamaModel):
token = None,
device_map = "sequential",
rope_scaling = None, # Mistral does not support RoPE scaling
check_tokenizer = True,
):
if rope_scaling is not None:
logger.warning_once("Unsloth: Mistral models do not support RoPE scaling.")
@ -332,14 +333,16 @@ class FastMistralModel(FastLlamaModel):
internal_model.max_seq_length = max_position_embeddings
# We check the tokenizer first for errors
tokenizer = check_tokenizer(
model = model,
tokenizer = tokenizer,
model_name = model_name,
model_max_length = max_seq_length,
padding_side = "right",
token = token,
)
if check_tokenizer:
tokenizer = check_tokenizer(
model = model,
tokenizer = tokenizer,
model_name = model_name,
model_max_length = max_seq_length,
padding_side = "right",
token = token,
)
pass
return model, tokenizer
pass
pass