fix: remove benchmark scripts from git tracking
These are standalone benchmark scripts that were force-added despite being gitignored. They have no test functions and run network calls at module level, which breaks pytest collection in CI.
This commit is contained in:
Roland Tannous
parent
9ca45826d4
commit
657cdaa151
3 changed files with 0 additions and 265 deletions
|
|
@ -1,54 +0,0 @@
|
|||
"""
|
||||
Benchmark: fsspec URL image download throughput at different dataset sizes.
|
||||
Dataset: google-research-datasets/conceptual_captions (subset: labeled)
|
||||
|
||||
Tests sizes: 100, 200, 300, 500, 1000, 1500, 2000
|
||||
Reports: time, success/fail rate, throughput (images/sec)
|
||||
"""
|
||||
from datasets import load_dataset, Dataset
|
||||
from PIL import Image as PILImage
|
||||
from io import BytesIO
|
||||
from itertools import islice
|
||||
import fsspec
|
||||
import time
|
||||
|
||||
DATASET = "google-research-datasets/conceptual_captions"
|
||||
SUBSET = "labeled"
|
||||
SPLIT = "train"
|
||||
SIZES = [100, 200, 300, 500, 1000, 1500, 2000]
|
||||
|
||||
# Load the max we need in one go
|
||||
max_size = max(SIZES)
|
||||
print(f"Loading {max_size} samples from {DATASET} (streaming)...")
|
||||
ds = load_dataset(DATASET, name=SUBSET, split=SPLIT, streaming=True)
|
||||
rows = list(islice(ds, max_size))
|
||||
full_dataset = Dataset.from_list(rows)
|
||||
print(f"Loaded {len(full_dataset)} samples")
|
||||
print(f"Columns: {full_dataset.column_names}")
|
||||
print()
|
||||
|
||||
print(f"{'Size':>6} | {'Time':>8} | {'OK':>6} | {'Fail':>6} | {'Fail%':>6} | {'img/s':>7}")
|
||||
print("-" * 55)
|
||||
|
||||
for size in SIZES:
|
||||
dataset = full_dataset.select(range(size))
|
||||
success, fail = 0, 0
|
||||
t0 = time.time()
|
||||
|
||||
for sample in dataset:
|
||||
url = sample["image_url"]
|
||||
try:
|
||||
with fsspec.open(url, "rb", expand=True) as f:
|
||||
img = PILImage.open(BytesIO(f.read())).convert("RGB")
|
||||
success += 1
|
||||
except Exception:
|
||||
fail += 1
|
||||
|
||||
elapsed = time.time() - t0
|
||||
fail_pct = (fail / size) * 100
|
||||
throughput = success / elapsed if elapsed > 0 else 0
|
||||
|
||||
print(f"{size:>6} | {elapsed:>7.1f}s | {success:>6} | {fail:>6} | {fail_pct:>5.1f}% | {throughput:>6.1f}/s")
|
||||
|
||||
print()
|
||||
print("Done.")
|
||||
|
|
@ -1,132 +0,0 @@
|
|||
"""
|
||||
Reproduce: VLM URL image loading with HF datasets.
|
||||
Tests cast_column(Image()) vs manual download approaches.
|
||||
Dataset: google-research-datasets/conceptual_captions (subset: labeled)
|
||||
"""
|
||||
from datasets import load_dataset, Image as datasets_Image, Dataset
|
||||
from PIL import Image as PILImage
|
||||
from io import BytesIO
|
||||
from itertools import islice
|
||||
import time
|
||||
|
||||
DATASET = "google-research-datasets/conceptual_captions"
|
||||
SUBSET = "labeled"
|
||||
SPLIT = "train"
|
||||
N_SAMPLES = 20 # small slice for testing
|
||||
|
||||
print("=" * 60)
|
||||
print("Loading dataset (streaming, first N samples)...")
|
||||
print("=" * 60)
|
||||
ds = load_dataset(DATASET, name=SUBSET, split=SPLIT, streaming=True)
|
||||
rows = list(islice(ds, N_SAMPLES))
|
||||
dataset = Dataset.from_list(rows)
|
||||
|
||||
print(f"Loaded {len(dataset)} samples")
|
||||
print(f"Columns: {dataset.column_names}")
|
||||
print(f"First image_url: {dataset[0]['image_url'][:100]}...")
|
||||
print()
|
||||
|
||||
# ─── Test 1: cast_column(Image()) — what we tried ───
|
||||
print("=" * 60)
|
||||
print("TEST 1: cast_column(Image()) approach")
|
||||
print("=" * 60)
|
||||
try:
|
||||
ds_cast = dataset.cast_column("image_url", datasets_Image())
|
||||
success, fail = 0, 0
|
||||
t0 = time.time()
|
||||
for i, sample in enumerate(ds_cast):
|
||||
try:
|
||||
img = sample["image_url"]
|
||||
if img is not None:
|
||||
print(f" [{i}] OK — {img.size} {img.mode}")
|
||||
success += 1
|
||||
else:
|
||||
print(f" [{i}] None returned")
|
||||
fail += 1
|
||||
except Exception as e:
|
||||
print(f" [{i}] FAILED: {type(e).__name__}: {str(e)[:80]}")
|
||||
fail += 1
|
||||
elapsed = time.time() - t0
|
||||
print(f"\nResult: {success} ok, {fail} failed, {elapsed:.1f}s")
|
||||
except Exception as e:
|
||||
print(f"CRASHED during iteration: {type(e).__name__}: {str(e)[:120]}")
|
||||
print()
|
||||
|
||||
# ─── Test 2: Manual download with requests.Session ───
|
||||
print("=" * 60)
|
||||
print("TEST 2: requests.Session() approach")
|
||||
print("=" * 60)
|
||||
try:
|
||||
import requests
|
||||
session = requests.Session()
|
||||
success, fail = 0, 0
|
||||
t0 = time.time()
|
||||
for i, sample in enumerate(dataset):
|
||||
url = sample["image_url"]
|
||||
try:
|
||||
resp = session.get(url, timeout=10)
|
||||
resp.raise_for_status()
|
||||
img = PILImage.open(BytesIO(resp.content)).convert("RGB")
|
||||
print(f" [{i}] OK — {img.size} {img.mode}")
|
||||
success += 1
|
||||
except Exception as e:
|
||||
print(f" [{i}] FAILED: {type(e).__name__}: {str(e)[:80]}")
|
||||
fail += 1
|
||||
elapsed = time.time() - t0
|
||||
print(f"\nResult: {success} ok, {fail} failed, {elapsed:.1f}s")
|
||||
except Exception as e:
|
||||
print(f"CRASHED: {type(e).__name__}: {str(e)[:120]}")
|
||||
print()
|
||||
|
||||
# ─── Test 3: urllib (stdlib) ───
|
||||
print("=" * 60)
|
||||
print("TEST 3: urllib approach (stdlib)")
|
||||
print("=" * 60)
|
||||
try:
|
||||
from urllib.request import urlopen, Request
|
||||
success, fail = 0, 0
|
||||
t0 = time.time()
|
||||
for i, sample in enumerate(dataset):
|
||||
url = sample["image_url"]
|
||||
try:
|
||||
req = Request(url, headers={"User-Agent": "Mozilla/5.0"})
|
||||
with urlopen(req, timeout=10) as resp:
|
||||
img = PILImage.open(BytesIO(resp.read())).convert("RGB")
|
||||
print(f" [{i}] OK — {img.size} {img.mode}")
|
||||
success += 1
|
||||
except Exception as e:
|
||||
print(f" [{i}] FAILED: {type(e).__name__}: {str(e)[:80]}")
|
||||
fail += 1
|
||||
elapsed = time.time() - t0
|
||||
print(f"\nResult: {success} ok, {fail} failed, {elapsed:.1f}s")
|
||||
except Exception as e:
|
||||
print(f"CRASHED: {type(e).__name__}: {str(e)[:120]}")
|
||||
print()
|
||||
|
||||
# ─── Test 4: fsspec directly with expand=True ───
|
||||
print("=" * 60)
|
||||
print("TEST 4: fsspec.open() with expand=True")
|
||||
print("=" * 60)
|
||||
try:
|
||||
import fsspec
|
||||
success, fail = 0, 0
|
||||
t0 = time.time()
|
||||
for i, sample in enumerate(dataset):
|
||||
url = sample["image_url"]
|
||||
try:
|
||||
with fsspec.open(url, "rb", expand=True) as f:
|
||||
img = PILImage.open(BytesIO(f.read())).convert("RGB")
|
||||
print(f" [{i}] OK — {img.size} {img.mode}")
|
||||
success += 1
|
||||
except Exception as e:
|
||||
print(f" [{i}] FAILED: {type(e).__name__}: {str(e)[:80]}")
|
||||
fail += 1
|
||||
elapsed = time.time() - t0
|
||||
print(f"\nResult: {success} ok, {fail} failed, {elapsed:.1f}s")
|
||||
except Exception as e:
|
||||
print(f"CRASHED: {type(e).__name__}: {str(e)[:120]}")
|
||||
|
||||
print()
|
||||
print("=" * 60)
|
||||
print("DONE — compare success rates and timing above")
|
||||
print("=" * 60)
|
||||
|
|
@ -1,79 +0,0 @@
|
|||
"""
|
||||
Benchmark: parallel fsspec URL image downloads with ThreadPoolExecutor.
|
||||
Tests different worker counts to find optimal parallelism.
|
||||
Dataset: google-research-datasets/conceptual_captions (subset: labeled)
|
||||
"""
|
||||
from datasets import load_dataset, Dataset
|
||||
from PIL import Image as PILImage
|
||||
from io import BytesIO
|
||||
from itertools import islice
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
import fsspec
|
||||
import time
|
||||
import os
|
||||
|
||||
DATASET = "google-research-datasets/conceptual_captions"
|
||||
SUBSET = "labeled"
|
||||
SPLIT = "train"
|
||||
N_SAMPLES = 500
|
||||
|
||||
# safe_num_proc formula from studio/backend/utils/hardware/hardware.py
|
||||
cpu_count = os.cpu_count()
|
||||
safe_workers = max(1, cpu_count // 3)
|
||||
print(f"CPU count: {cpu_count}, safe_num_proc: {safe_workers}")
|
||||
|
||||
WORKER_COUNTS = [1, 4, 8, 16, 32, safe_workers]
|
||||
# Deduplicate and sort
|
||||
WORKER_COUNTS = sorted(set(WORKER_COUNTS))
|
||||
|
||||
print(f"Loading {N_SAMPLES} samples from {DATASET} (streaming)...")
|
||||
ds = load_dataset(DATASET, name=SUBSET, split=SPLIT, streaming=True)
|
||||
rows = list(islice(ds, N_SAMPLES))
|
||||
dataset = Dataset.from_list(rows)
|
||||
urls = [row["image_url"] for row in dataset]
|
||||
print(f"Loaded {len(urls)} URLs")
|
||||
print()
|
||||
|
||||
|
||||
def download_single(url):
|
||||
"""Download a single image URL using fsspec. Returns PIL image or raises."""
|
||||
with fsspec.open(url, "rb", expand=True) as f:
|
||||
img = PILImage.open(BytesIO(f.read())).convert("RGB")
|
||||
return img
|
||||
|
||||
|
||||
print(f"{'Workers':>8} | {'Time':>8} | {'OK':>6} | {'Fail':>6} | {'Fail%':>6} | {'img/s':>7} | {'Speedup':>8}")
|
||||
print("-" * 70)
|
||||
|
||||
baseline_throughput = None
|
||||
|
||||
for n_workers in WORKER_COUNTS:
|
||||
success, fail = 0, 0
|
||||
t0 = time.time()
|
||||
|
||||
with ThreadPoolExecutor(max_workers=n_workers) as pool:
|
||||
futures = {pool.submit(download_single, url): url for url in urls}
|
||||
for future in as_completed(futures):
|
||||
try:
|
||||
img = future.result(timeout=30)
|
||||
success += 1
|
||||
except Exception:
|
||||
fail += 1
|
||||
|
||||
elapsed = time.time() - t0
|
||||
fail_pct = (fail / N_SAMPLES) * 100
|
||||
throughput = success / elapsed if elapsed > 0 else 0
|
||||
|
||||
if baseline_throughput is None:
|
||||
baseline_throughput = throughput
|
||||
speedup = throughput / baseline_throughput if baseline_throughput > 0 else 0
|
||||
|
||||
label = f"{n_workers}"
|
||||
if n_workers == safe_workers:
|
||||
label += "*" # mark the safe_num_proc value
|
||||
|
||||
print(f"{label:>8} | {elapsed:>7.1f}s | {success:>6} | {fail:>6} | {fail_pct:>5.1f}% | {throughput:>6.1f}/s | {speedup:>7.1f}x")
|
||||
|
||||
print()
|
||||
print("* = safe_num_proc value")
|
||||
print("Done.")
|
||||
Loading…
Add table
Add a link
Reference in a new issue