ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[pre-commit.ci] auto fixes from pre-commit.com hooks

Browse source 
for more information, see https://pre-commit.ci
This commit is contained in:
pre-commit-ci[bot] 2025-12-01 13:33:35 +00:00
commit fbb98c5c5c
47 changed files with 2646 additions and 2646 deletions
Download patch file Download diff file Expand all files Collapse all files

6
tests/utils/perplexity_eval.py
View file

@ -12,7 +12,7 @@ def ppl_model(model, tokenizer, dataset):
max_length = 2048
stride = 512
for s in tqdm(range(len(dataset["text"]))):
encodings = tokenizer(dataset["text"][s], return_tensors = "pt")
encodings = tokenizer(dataset["text"][s], return_tensors="pt")
seq_len = encodings.input_ids.size(1)
prev_end_loc = 0
for begin_loc in range(0, seq_len, stride):
@ -28,7 +28,7 @@ def ppl_model(model, tokenizer, dataset):
attention_mask = (input_ids != pad_token_id).long()
with torch.no_grad():
outputs = model(
input_ids, labels = target_ids, attention_mask = attention_mask
input_ids, labels=target_ids, attention_mask=attention_mask
)
neg_log_likelihood = outputs.loss
nlls.append(neg_log_likelihood)
@ -78,4 +78,4 @@ def print_model_comparison():
# Display the comparison table
print("\nComparison Table:")
print(comparison_df.to_string(index = False))
print(comparison_df.to_string(index=False))