e662af769b* fix: add XPU device support and update hardcoded CUDA selections * fix: add XPU device support for pytest CUDA skipped tests * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix device handling for PR #7401 - perplexity_eval.py: use DEVICE_TYPE_TORCH, not DEVICE_TYPE. The latter can be "hip" or "mlx", which .to() rejects, so this regressed ROCm. - test_batched_leftpad_generation_gpu.py: XPU diverges here today, so mark it non-strict xfail on XPU instead of reverting to a CUDA-only guard. Keeps the real XPU gap visible and turns green once it is fixed. - Guard torch.xpu.is_available() with hasattr, matching device_type.py. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Re-enable the flash varlen attention test in CI for PR #7401 attention_dispatch.py now predefines flash_attn_func / flash_attn_varlen_func as None, so test_run_attention_flash_varlen_receives_window_and_softcap no longer needs flash_attn importable to be monkeypatched. Verified on a runner shaped like the CPU-only one: the test fails against main's attention_dispatch and passes at this head, so the deselect is now dead weight. * Tighten comments for PR #7401 Drop the hasattr rationale: torch.xpu has existed since torch 2.3 and the dependency floor is 2.4, so no supported build predates the namespace. The guard stays as cheap defence, but the comment claimed something untrue. --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: danielhanchen <danielhanchen@gmail.com>
67 lines
2.3 KiB
Python
67 lines
2.3 KiB
Python
from tqdm import tqdm
|
|
import torch
|
|
import pandas as pd
|
|
|
|
# DEVICE_TYPE_TORCH, not DEVICE_TYPE: the latter can be "hip"/"mlx", which .to() rejects.
|
|
from unsloth.device_type import DEVICE_TYPE_TORCH
|
|
|
|
model_comparison_results = {}
|
|
|
|
|
|
# Per-example perplexity, sliding window for examples longer than 512 tokens.
|
|
def ppl_model(model, tokenizer, dataset):
|
|
nlls = []
|
|
max_length = 2048
|
|
stride = 512
|
|
for s in tqdm(range(len(dataset["text"]))):
|
|
encodings = tokenizer(dataset["text"][s], return_tensors = "pt")
|
|
seq_len = encodings.input_ids.size(1)
|
|
prev_end_loc = 0
|
|
for begin_loc in range(0, seq_len, stride):
|
|
end_loc = min(begin_loc + max_length, seq_len)
|
|
trg_len = end_loc - prev_end_loc
|
|
input_ids = encodings.input_ids[:, begin_loc:end_loc].to(DEVICE_TYPE_TORCH)
|
|
target_ids = input_ids.clone()
|
|
target_ids[:, :-trg_len] = -100
|
|
pad_token_id = tokenizer.pad_token_id if tokenizer.pad_token_id is not None else 0
|
|
attention_mask = (input_ids != pad_token_id).long()
|
|
with torch.no_grad():
|
|
outputs = model(input_ids, labels = target_ids, attention_mask = attention_mask)
|
|
neg_log_likelihood = outputs.loss
|
|
nlls.append(neg_log_likelihood)
|
|
prev_end_loc = end_loc
|
|
if end_loc == seq_len:
|
|
break
|
|
ppl = torch.exp(torch.stack(nlls).mean())
|
|
return ppl
|
|
|
|
|
|
# ----------- Reporting helpers ----------- #
|
|
|
|
|
|
def add_to_comparison(model_name, ppl):
|
|
"""Record a model's perplexity in the comparison tracker."""
|
|
model_comparison_results[model_name] = {"ppl": ppl}
|
|
|
|
|
|
def print_model_comparison():
|
|
"""Print a comparison of all models evaluated so far"""
|
|
if not model_comparison_results:
|
|
print("No model results available for comparison")
|
|
return
|
|
|
|
print("\n==== MODEL COMPARISON REPORT ====")
|
|
|
|
comparison_df = pd.DataFrame(
|
|
{
|
|
"Model": list(model_comparison_results.keys()),
|
|
"Perplexity": [
|
|
# Tensors to CPU float if needed.
|
|
results["ppl"].cpu().item() if torch.is_tensor(results["ppl"]) else results["ppl"]
|
|
for results in model_comparison_results.values()
|
|
],
|
|
}
|
|
)
|
|
|
|
print("\nComparison Table:")
|
|
print(comparison_df.to_string(index = False))
|