From 657cdaa1514c0151de98ee1c2f837f36cc5beb15 Mon Sep 17 00:00:00 2001 From: Roland Tannous Date: Thu, 5 Mar 2026 06:06:47 +0000 Subject: [PATCH] fix: remove benchmark scripts from git tracking These are standalone benchmark scripts that were force-added despite being gitignored. They have no test functions and run network calls at module level, which breaks pytest collection in CI. --- studio/tests/test_url_download_benchmark.py | 54 -------- studio/tests/test_url_image_loading.py | 132 -------------------- studio/tests/test_url_parallel_benchmark.py | 79 ------------ 3 files changed, 265 deletions(-) delete mode 100644 studio/tests/test_url_download_benchmark.py delete mode 100644 studio/tests/test_url_image_loading.py delete mode 100644 studio/tests/test_url_parallel_benchmark.py diff --git a/studio/tests/test_url_download_benchmark.py b/studio/tests/test_url_download_benchmark.py deleted file mode 100644 index e29a8e05b6..0000000000 --- a/studio/tests/test_url_download_benchmark.py +++ /dev/null @@ -1,54 +0,0 @@ -""" -Benchmark: fsspec URL image download throughput at different dataset sizes. -Dataset: google-research-datasets/conceptual_captions (subset: labeled) - -Tests sizes: 100, 200, 300, 500, 1000, 1500, 2000 -Reports: time, success/fail rate, throughput (images/sec) -""" -from datasets import load_dataset, Dataset -from PIL import Image as PILImage -from io import BytesIO -from itertools import islice -import fsspec -import time - -DATASET = "google-research-datasets/conceptual_captions" -SUBSET = "labeled" -SPLIT = "train" -SIZES = [100, 200, 300, 500, 1000, 1500, 2000] - -# Load the max we need in one go -max_size = max(SIZES) -print(f"Loading {max_size} samples from {DATASET} (streaming)...") -ds = load_dataset(DATASET, name=SUBSET, split=SPLIT, streaming=True) -rows = list(islice(ds, max_size)) -full_dataset = Dataset.from_list(rows) -print(f"Loaded {len(full_dataset)} samples") -print(f"Columns: {full_dataset.column_names}") -print() - -print(f"{'Size':>6} | {'Time':>8} | {'OK':>6} | {'Fail':>6} | {'Fail%':>6} | {'img/s':>7}") -print("-" * 55) - -for size in SIZES: - dataset = full_dataset.select(range(size)) - success, fail = 0, 0 - t0 = time.time() - - for sample in dataset: - url = sample["image_url"] - try: - with fsspec.open(url, "rb", expand=True) as f: - img = PILImage.open(BytesIO(f.read())).convert("RGB") - success += 1 - except Exception: - fail += 1 - - elapsed = time.time() - t0 - fail_pct = (fail / size) * 100 - throughput = success / elapsed if elapsed > 0 else 0 - - print(f"{size:>6} | {elapsed:>7.1f}s | {success:>6} | {fail:>6} | {fail_pct:>5.1f}% | {throughput:>6.1f}/s") - -print() -print("Done.") diff --git a/studio/tests/test_url_image_loading.py b/studio/tests/test_url_image_loading.py deleted file mode 100644 index 4899bab8e3..0000000000 --- a/studio/tests/test_url_image_loading.py +++ /dev/null @@ -1,132 +0,0 @@ -""" -Reproduce: VLM URL image loading with HF datasets. -Tests cast_column(Image()) vs manual download approaches. -Dataset: google-research-datasets/conceptual_captions (subset: labeled) -""" -from datasets import load_dataset, Image as datasets_Image, Dataset -from PIL import Image as PILImage -from io import BytesIO -from itertools import islice -import time - -DATASET = "google-research-datasets/conceptual_captions" -SUBSET = "labeled" -SPLIT = "train" -N_SAMPLES = 20 # small slice for testing - -print("=" * 60) -print("Loading dataset (streaming, first N samples)...") -print("=" * 60) -ds = load_dataset(DATASET, name=SUBSET, split=SPLIT, streaming=True) -rows = list(islice(ds, N_SAMPLES)) -dataset = Dataset.from_list(rows) - -print(f"Loaded {len(dataset)} samples") -print(f"Columns: {dataset.column_names}") -print(f"First image_url: {dataset[0]['image_url'][:100]}...") -print() - -# ─── Test 1: cast_column(Image()) — what we tried ─── -print("=" * 60) -print("TEST 1: cast_column(Image()) approach") -print("=" * 60) -try: - ds_cast = dataset.cast_column("image_url", datasets_Image()) - success, fail = 0, 0 - t0 = time.time() - for i, sample in enumerate(ds_cast): - try: - img = sample["image_url"] - if img is not None: - print(f" [{i}] OK — {img.size} {img.mode}") - success += 1 - else: - print(f" [{i}] None returned") - fail += 1 - except Exception as e: - print(f" [{i}] FAILED: {type(e).__name__}: {str(e)[:80]}") - fail += 1 - elapsed = time.time() - t0 - print(f"\nResult: {success} ok, {fail} failed, {elapsed:.1f}s") -except Exception as e: - print(f"CRASHED during iteration: {type(e).__name__}: {str(e)[:120]}") -print() - -# ─── Test 2: Manual download with requests.Session ─── -print("=" * 60) -print("TEST 2: requests.Session() approach") -print("=" * 60) -try: - import requests - session = requests.Session() - success, fail = 0, 0 - t0 = time.time() - for i, sample in enumerate(dataset): - url = sample["image_url"] - try: - resp = session.get(url, timeout=10) - resp.raise_for_status() - img = PILImage.open(BytesIO(resp.content)).convert("RGB") - print(f" [{i}] OK — {img.size} {img.mode}") - success += 1 - except Exception as e: - print(f" [{i}] FAILED: {type(e).__name__}: {str(e)[:80]}") - fail += 1 - elapsed = time.time() - t0 - print(f"\nResult: {success} ok, {fail} failed, {elapsed:.1f}s") -except Exception as e: - print(f"CRASHED: {type(e).__name__}: {str(e)[:120]}") -print() - -# ─── Test 3: urllib (stdlib) ─── -print("=" * 60) -print("TEST 3: urllib approach (stdlib)") -print("=" * 60) -try: - from urllib.request import urlopen, Request - success, fail = 0, 0 - t0 = time.time() - for i, sample in enumerate(dataset): - url = sample["image_url"] - try: - req = Request(url, headers={"User-Agent": "Mozilla/5.0"}) - with urlopen(req, timeout=10) as resp: - img = PILImage.open(BytesIO(resp.read())).convert("RGB") - print(f" [{i}] OK — {img.size} {img.mode}") - success += 1 - except Exception as e: - print(f" [{i}] FAILED: {type(e).__name__}: {str(e)[:80]}") - fail += 1 - elapsed = time.time() - t0 - print(f"\nResult: {success} ok, {fail} failed, {elapsed:.1f}s") -except Exception as e: - print(f"CRASHED: {type(e).__name__}: {str(e)[:120]}") -print() - -# ─── Test 4: fsspec directly with expand=True ─── -print("=" * 60) -print("TEST 4: fsspec.open() with expand=True") -print("=" * 60) -try: - import fsspec - success, fail = 0, 0 - t0 = time.time() - for i, sample in enumerate(dataset): - url = sample["image_url"] - try: - with fsspec.open(url, "rb", expand=True) as f: - img = PILImage.open(BytesIO(f.read())).convert("RGB") - print(f" [{i}] OK — {img.size} {img.mode}") - success += 1 - except Exception as e: - print(f" [{i}] FAILED: {type(e).__name__}: {str(e)[:80]}") - fail += 1 - elapsed = time.time() - t0 - print(f"\nResult: {success} ok, {fail} failed, {elapsed:.1f}s") -except Exception as e: - print(f"CRASHED: {type(e).__name__}: {str(e)[:120]}") - -print() -print("=" * 60) -print("DONE — compare success rates and timing above") -print("=" * 60) diff --git a/studio/tests/test_url_parallel_benchmark.py b/studio/tests/test_url_parallel_benchmark.py deleted file mode 100644 index a0d160136c..0000000000 --- a/studio/tests/test_url_parallel_benchmark.py +++ /dev/null @@ -1,79 +0,0 @@ -""" -Benchmark: parallel fsspec URL image downloads with ThreadPoolExecutor. -Tests different worker counts to find optimal parallelism. -Dataset: google-research-datasets/conceptual_captions (subset: labeled) -""" -from datasets import load_dataset, Dataset -from PIL import Image as PILImage -from io import BytesIO -from itertools import islice -from concurrent.futures import ThreadPoolExecutor, as_completed -import fsspec -import time -import os - -DATASET = "google-research-datasets/conceptual_captions" -SUBSET = "labeled" -SPLIT = "train" -N_SAMPLES = 500 - -# safe_num_proc formula from studio/backend/utils/hardware/hardware.py -cpu_count = os.cpu_count() -safe_workers = max(1, cpu_count // 3) -print(f"CPU count: {cpu_count}, safe_num_proc: {safe_workers}") - -WORKER_COUNTS = [1, 4, 8, 16, 32, safe_workers] -# Deduplicate and sort -WORKER_COUNTS = sorted(set(WORKER_COUNTS)) - -print(f"Loading {N_SAMPLES} samples from {DATASET} (streaming)...") -ds = load_dataset(DATASET, name=SUBSET, split=SPLIT, streaming=True) -rows = list(islice(ds, N_SAMPLES)) -dataset = Dataset.from_list(rows) -urls = [row["image_url"] for row in dataset] -print(f"Loaded {len(urls)} URLs") -print() - - -def download_single(url): - """Download a single image URL using fsspec. Returns PIL image or raises.""" - with fsspec.open(url, "rb", expand=True) as f: - img = PILImage.open(BytesIO(f.read())).convert("RGB") - return img - - -print(f"{'Workers':>8} | {'Time':>8} | {'OK':>6} | {'Fail':>6} | {'Fail%':>6} | {'img/s':>7} | {'Speedup':>8}") -print("-" * 70) - -baseline_throughput = None - -for n_workers in WORKER_COUNTS: - success, fail = 0, 0 - t0 = time.time() - - with ThreadPoolExecutor(max_workers=n_workers) as pool: - futures = {pool.submit(download_single, url): url for url in urls} - for future in as_completed(futures): - try: - img = future.result(timeout=30) - success += 1 - except Exception: - fail += 1 - - elapsed = time.time() - t0 - fail_pct = (fail / N_SAMPLES) * 100 - throughput = success / elapsed if elapsed > 0 else 0 - - if baseline_throughput is None: - baseline_throughput = throughput - speedup = throughput / baseline_throughput if baseline_throughput > 0 else 0 - - label = f"{n_workers}" - if n_workers == safe_workers: - label += "*" # mark the safe_num_proc value - - print(f"{label:>8} | {elapsed:>7.1f}s | {success:>6} | {fail:>6} | {fail_pct:>5.1f}% | {throughput:>6.1f}/s | {speedup:>7.1f}x") - -print() -print("* = safe_num_proc value") -print("Done.")