ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix_tokenizer

Browse source
This commit is contained in:
Daniel Han-Chen 2024-01-10 01:02:44 +11:00
commit 82e6fece0b
3 changed files with 6 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

4
unsloth/models/llama.py
View file

@ -619,7 +619,7 @@ class FastLlamaModel:
token = None,
device_map = "sequential",
rope_scaling = None,
check_tokenizer = True,
fix_tokenizer = True,
):
SUPPORTS_BFLOAT16 = torch.cuda.is_bf16_supported()
gpu_stats = torch.cuda.get_device_properties(0)
@ -704,7 +704,7 @@ class FastLlamaModel:
internal_model.max_seq_length = max_position_embeddings
# We check the tokenizer first for errors
if check_tokenizer:
if fix_tokenizer:
tokenizer = check_tokenizer(
model = model,
tokenizer = tokenizer,

4
unsloth/models/loader.py
View file

@ -44,7 +44,7 @@ class FastLanguageModel(FastLlamaModel):
token = None,
device_map = "sequential",
rope_scaling = None,
check_tokenizer = True,
fix_tokenizer = True,
*args, **kwargs,
):
if not SUPPORTS_FOURBIT and model_name in FOURBIT_MAPPER:
@ -84,7 +84,7 @@ class FastLanguageModel(FastLlamaModel):
token = token,
device_map = device_map,
rope_scaling = rope_scaling,
check_tokenizer = check_tokenizer,
fix_tokenizer = fix_tokenizer,
*args, **kwargs,
)
pass

4
unsloth/models/mistral.py
View file

@ -263,7 +263,7 @@ class FastMistralModel(FastLlamaModel):
token = None,
device_map = "sequential",
rope_scaling = None, # Mistral does not support RoPE scaling
check_tokenizer = True,
fix_tokenizer = True,
):
if rope_scaling is not None:
logger.warning_once("Unsloth: Mistral models do not support RoPE scaling.")
@ -333,7 +333,7 @@ class FastMistralModel(FastLlamaModel):
internal_model.max_seq_length = max_position_embeddings
# We check the tokenizer first for errors
if check_tokenizer:
if fix_tokenizer:
tokenizer = check_tokenizer(
model = model,
tokenizer = tokenizer,