From 91783c0fb29cba95354c7d5b43a24d3e8ef6e394 Mon Sep 17 00:00:00 2001 From: Roland Tannous <115670425+rolandtannous@users.noreply.github.com> Date: Thu, 5 Mar 2026 03:21:07 +0400 Subject: [PATCH] Revert "Add index range dataset slicing to Studio training page" --- studio/backend/core/training/trainer.py | 24 +-- studio/backend/core/training/training.py | 6 +- studio/backend/models/training.py | 2 - studio/backend/routes/training.py | 2 - .../utils/datasets/format_conversion.py | 64 +------ .../studio/sections/dataset-section.tsx | 166 +++++------------- .../src/features/training/api/mappers.ts | 11 -- .../training/stores/training-config-store.ts | 12 +- .../src/features/training/types/api.ts | 2 - .../src/features/training/types/config.ts | 4 - studio/tests/test_url_download_benchmark.py | 54 ------ studio/tests/test_url_image_loading.py | 132 -------------- studio/tests/test_url_parallel_benchmark.py | 79 --------- 13 files changed, 55 insertions(+), 503 deletions(-) delete mode 100644 studio/tests/test_url_download_benchmark.py delete mode 100644 studio/tests/test_url_image_loading.py delete mode 100644 studio/tests/test_url_parallel_benchmark.py diff --git a/studio/backend/core/training/trainer.py b/studio/backend/core/training/trainer.py index f5b2f18245..e4e7a474be 100644 --- a/studio/backend/core/training/trainer.py +++ b/studio/backend/core/training/trainer.py @@ -353,9 +353,7 @@ class UnslothTrainer: subset: str = None, train_split: str = "train", eval_split: str = None, - eval_steps: float = 0.00, - dataset_slice_start: int = None, - dataset_slice_end: int = None) -> Optional[tuple]: + eval_steps: float = 0.00) -> Optional[tuple]: """ Load and prepare dataset for training. @@ -447,18 +445,6 @@ class UnslothTrainer: if dataset is None: raise ValueError("No dataset provided") - # Apply index range slicing if requested (inclusive on both ends) - if dataset_slice_start is not None or dataset_slice_end is not None: - total_rows = len(dataset) - start = dataset_slice_start if dataset_slice_start is not None else 0 - end = dataset_slice_end if dataset_slice_end is not None else total_rows - 1 - # Clamp to valid range - start = max(0, min(start, total_rows - 1)) - end = max(start, min(end, total_rows - 1)) - dataset = dataset.select(range(start, end + 1)) - print(f"Sliced dataset to rows [{start}, {end}]: {len(dataset)} of {total_rows} rows\n") - self._update_progress(status_message=f"Sliced dataset to {len(dataset)} rows (indices {start}-{end})") - # Check if stopped before applying template if self.should_stop: print("Stopped before applying chat template\n") @@ -482,14 +468,6 @@ class UnslothTrainer: print("Stopped during dataset formatting\n") return None - # Abort if dataset formatting/conversion failed - if not dataset_info.get("success", True): - errors = dataset_info.get("errors", []) - error_msg = "; ".join(errors) if errors else "Dataset formatting failed" - logger.error(f"Dataset conversion failed: {error_msg}") - self._update_progress(error=error_msg) - return None - self._update_progress(status_message=f"Dataset formatted and ready for training") print(f"Dataset formatted successfully\n") diff --git a/studio/backend/core/training/training.py b/studio/backend/core/training/training.py index 153f4335e3..9123d36b39 100644 --- a/studio/backend/core/training/training.py +++ b/studio/backend/core/training/training.py @@ -116,9 +116,7 @@ class TrainingBackend: train_split: str = "train", eval_split: str = None, eval_steps: float = 0.00, - is_dataset_multimodal: bool = False, - dataset_slice_start: int = None, - dataset_slice_end: int = None) -> bool: + is_dataset_multimodal: bool = False) -> bool: """ Start training. @@ -226,8 +224,6 @@ class TrainingBackend: train_split=train_split, eval_split=eval_split, eval_steps=eval_steps, - dataset_slice_start=dataset_slice_start, - dataset_slice_end=dataset_slice_end, ) # Unpack: load_and_format_dataset returns (dataset, eval_dataset) diff --git a/studio/backend/models/training.py b/studio/backend/models/training.py index b6b30989bd..54de974100 100644 --- a/studio/backend/models/training.py +++ b/studio/backend/models/training.py @@ -22,8 +22,6 @@ class TrainingStartRequest(BaseModel): train_split: Optional[str] = Field("train", description="Training split name") eval_split: Optional[str] = Field(None, description="Eval split name. None = auto-detect") eval_steps: float = Field(0.00, description="Fraction of total steps between evals (0-1)") - dataset_slice_start: Optional[int] = Field(None, description="Inclusive start row index for dataset slicing") - dataset_slice_end: Optional[int] = Field(None, description="Inclusive end row index for dataset slicing") @model_validator(mode="before") @classmethod diff --git a/studio/backend/routes/training.py b/studio/backend/routes/training.py index 497daaedd3..f8de2f639f 100644 --- a/studio/backend/routes/training.py +++ b/studio/backend/routes/training.py @@ -149,8 +149,6 @@ async def start_training( "train_split": request.train_split, "eval_split": request.eval_split, "eval_steps": request.eval_steps, - "dataset_slice_start": request.dataset_slice_start, - "dataset_slice_end": request.dataset_slice_end, "custom_format_mapping": request.custom_format_mapping, "num_epochs": request.num_epochs, "learning_rate": request.learning_rate, diff --git a/studio/backend/utils/datasets/format_conversion.py b/studio/backend/utils/datasets/format_conversion.py index e784e8e645..6436e7a82a 100644 --- a/studio/backend/utils/datasets/format_conversion.py +++ b/studio/backend/utils/datasets/format_conversion.py @@ -281,17 +281,12 @@ def convert_to_vlm_format( def _convert_single_sample(sample): """Convert a single sample to VLM format.""" - # Get image (might be PIL Image, local path, or URL) + # Get image (might be PIL Image or path) image_data = sample[image_column] + # Handle image paths if isinstance(image_data, str): - if image_data.startswith(("http://", "https://")): - import fsspec - from io import BytesIO - with fsspec.open(image_data, "rb", expand=True) as f: - image_data = Image.open(BytesIO(f.read())).convert("RGB") - else: - image_data = Image.open(image_data).convert("RGB") + image_data = Image.open(image_data).convert("RGB") # Get text text_data = sample[text_column] @@ -322,56 +317,11 @@ def convert_to_vlm_format( # Return dict with messages return {"messages": messages} - # Convert samples, skipping any with broken/unreachable images. - # For URL-based datasets, check the first PROBE_SIZE samples early to - # fail fast if too many images are broken, before downloading millions. - PROBE_SIZE = 5000 - MAX_FAIL_RATE = 0.3 + # Use list comprehension and return the LIST directly + print(f"🔄 Converting {len(dataset)} samples to VLM format...") + converted_list = [_convert_single_sample(sample) for sample in dataset] - total = len(dataset) - has_urls = isinstance(next(iter(dataset))[image_column], str) - probe_needed = has_urls and total > PROBE_SIZE - - from tqdm import tqdm - - print(f"🔄 Converting {total} samples to VLM format...") - converted_list = [] - failed_count = 0 - - pbar = tqdm(dataset, total=total, desc="Converting VLM samples", unit="sample") - for i, sample in enumerate(pbar): - try: - converted_list.append(_convert_single_sample(sample)) - except Exception as e: - failed_count += 1 - - pbar.set_postfix(ok=len(converted_list), failed=failed_count, refresh=False) - - # Early exit check after probing the first batch - if probe_needed and (i + 1) == PROBE_SIZE: - fail_rate = failed_count / PROBE_SIZE - if fail_rate >= MAX_FAIL_RATE: - pbar.close() - raise ValueError( - f"{fail_rate:.0%} of the first {PROBE_SIZE} images failed to download " - f"({failed_count}/{PROBE_SIZE}). " - "This dataset has too many broken or unreachable image URLs. " - "Consider using a dataset with embedded images instead." - ) - print(f"✅ Probe passed: {failed_count}/{PROBE_SIZE} ({fail_rate:.0%}) failures in first batch, continuing...") - pbar.close() - - if failed_count > 0: - fail_rate = failed_count / total - print(f"⚠️ Skipped {failed_count}/{total} ({fail_rate:.0%}) samples with broken/unreachable images") - - if len(converted_list) == 0: - raise ValueError( - f"All {total} samples failed during VLM conversion — no usable images found. " - "This dataset may contain only image URLs that are no longer accessible." - ) - - print(f"✅ Converted {len(converted_list)}/{total} samples") + print(f"✅ Converted {len(converted_list)} samples") # Return list, NOT Dataset return converted_list diff --git a/studio/frontend/src/features/studio/sections/dataset-section.tsx b/studio/frontend/src/features/studio/sections/dataset-section.tsx index 28254adcc7..57e50bced5 100644 --- a/studio/frontend/src/features/studio/sections/dataset-section.tsx +++ b/studio/frontend/src/features/studio/sections/dataset-section.tsx @@ -13,7 +13,6 @@ import { ComboboxItem, ComboboxList, } from "@/components/ui/combobox"; -import { Input } from "@/components/ui/input"; import { InputGroupAddon } from "@/components/ui/input-group"; import { Select, @@ -76,10 +75,6 @@ export function DatasetSection() { setDatasetEvalSplit, hfToken, modelType, - datasetSliceStart, - setDatasetSliceStart, - datasetSliceEnd, - setDatasetSliceEnd, } = useTrainingConfigStore( useShallow((s) => ({ dataset: s.dataset, @@ -94,10 +89,6 @@ export function DatasetSection() { setDatasetEvalSplit: s.setDatasetEvalSplit, hfToken: s.hfToken, modelType: s.modelType, - datasetSliceStart: s.datasetSliceStart, - setDatasetSliceStart: s.setDatasetSliceStart, - datasetSliceEnd: s.datasetSliceEnd, - setDatasetSliceEnd: s.setDatasetSliceEnd, })), ); @@ -302,118 +293,51 @@ export function DatasetSection() { Advanced -
-
- - Target Format - - - - - - Format of your training data. Auto-detect works for most - datasets.{" "} - - Read more - - - - - -
-
-
- - Train Split Start - - - - - - Only train on a subset of your training split by - specifying a start row index (inclusive, 0-based). - Leave empty to start from the first row. - - - - - setDatasetSliceStart(e.target.value || null) - } - /> -
-
- - Train Split End - - - - - - Last row index to include from the training split - (inclusive, 0-based). For example, set Start to 0 and - End to 99 to train on the first 100 rows. Leave empty - to use all remaining rows. - - - - - setDatasetSliceEnd(e.target.value || null) - } - /> -
-
+
+ + Target Format + + + + + + Format of your training data. Auto-detect works for most + datasets.{" "} + + Read more + + + + +
diff --git a/studio/frontend/src/features/training/api/mappers.ts b/studio/frontend/src/features/training/api/mappers.ts index cd0d1f14e1..1adfbd8b6d 100644 --- a/studio/frontend/src/features/training/api/mappers.ts +++ b/studio/frontend/src/features/training/api/mappers.ts @@ -4,15 +4,6 @@ import type { TrainingStartRequest } from "../types/api"; const BACKEND_LORA_TYPE = "LoRA/QLoRA"; const BACKEND_FULL_TYPE = "Full Finetuning"; -function parseSliceValue(value: string | null): number | null { - if (value == null) return null; - const trimmed = value.trim(); - if (!trimmed) return null; - const num = Number(trimmed); - if (!Number.isFinite(num) || !Number.isInteger(num)) return null; - return num; -} - export function toBackendTrainingType(trainingMethod: string): string { return trainingMethod === "full" ? BACKEND_FULL_TYPE : BACKEND_LORA_TYPE; } @@ -36,8 +27,6 @@ export function buildTrainingStartPayload( subset: hfDataset ? config.datasetSubset : null, train_split: hfDataset ? config.datasetSplit : null, eval_split: hfDataset ? config.datasetEvalSplit : null, - dataset_slice_start: parseSliceValue(config.datasetSliceStart), - dataset_slice_end: parseSliceValue(config.datasetSliceEnd), local_datasets: [], format_type: config.datasetFormat, custom_format_mapping: customFormatMapping, diff --git a/studio/frontend/src/features/training/stores/training-config-store.ts b/studio/frontend/src/features/training/stores/training-config-store.ts index 93c742ba98..b2d1858716 100644 --- a/studio/frontend/src/features/training/stores/training-config-store.ts +++ b/studio/frontend/src/features/training/stores/training-config-store.ts @@ -28,8 +28,6 @@ const initialState: TrainingConfigState = { datasetSplit: null, datasetEvalSplit: null, datasetManualMapping: emptyManualMapping(), - datasetSliceStart: null, - datasetSliceEnd: null, uploadedFile: null, isCheckingVision: false, isVisionModel: false, @@ -257,8 +255,6 @@ export const useTrainingConfigStore = create()( datasetSplit: null, datasetEvalSplit: null, datasetManualMapping: emptyManualMapping(), - datasetSliceStart: null, - datasetSliceEnd: null, isDatasetMultimodal: null, isCheckingDataset: false, }); @@ -315,8 +311,6 @@ export const useTrainingConfigStore = create()( }, setDatasetManualMapping: (datasetManualMapping) => set({ datasetManualMapping }), - setDatasetSliceStart: (datasetSliceStart) => set({ datasetSliceStart }), - setDatasetSliceEnd: (datasetSliceEnd) => set({ datasetSliceEnd }), setUploadedFile: (uploadedFile) => set({ uploadedFile }), setEpochs: (epochs) => set({ epochs }), setContextLength: (contextLength) => set({ contextLength }), @@ -374,7 +368,7 @@ export const useTrainingConfigStore = create()( }, { name: "unsloth_training_config_v1", - version: 7, + version: 6, migrate: (persisted, version) => { const s = persisted as Record; if (version < 2 && s.datasetSubset == null && s.datasetConfig != null) { @@ -393,10 +387,6 @@ export const useTrainingConfigStore = create()( if (version < 6 && s.datasetEvalSplit == null) { s.datasetEvalSplit = null; } - if (version < 7) { - s.datasetSliceStart ??= null; - s.datasetSliceEnd ??= null; - } return s as unknown as TrainingConfigStore; }, partialize: partializePersistedState, diff --git a/studio/frontend/src/features/training/types/api.ts b/studio/frontend/src/features/training/types/api.ts index e2fcc04ad4..22f02e4331 100644 --- a/studio/frontend/src/features/training/types/api.ts +++ b/studio/frontend/src/features/training/types/api.ts @@ -8,8 +8,6 @@ export interface TrainingStartRequest { subset: string | null; train_split: string | null; eval_split: string | null; - dataset_slice_start: number | null; - dataset_slice_end: number | null; local_datasets: string[]; format_type: string; custom_format_mapping?: Record | null; diff --git a/studio/frontend/src/features/training/types/config.ts b/studio/frontend/src/features/training/types/config.ts index 0e48d18861..6c2feec172 100644 --- a/studio/frontend/src/features/training/types/config.ts +++ b/studio/frontend/src/features/training/types/config.ts @@ -26,8 +26,6 @@ export interface TrainingConfigState { datasetSplit: string | null; datasetEvalSplit: string | null; datasetManualMapping: DatasetManualMapping; - datasetSliceStart: string | null; - datasetSliceEnd: string | null; uploadedFile: string | null; epochs: number; contextLength: number; @@ -86,8 +84,6 @@ export interface TrainingConfigActions { setDatasetSplit: (split: string | null) => void; setDatasetEvalSplit: (split: string | null) => void; setDatasetManualMapping: (mapping: DatasetManualMapping) => void; - setDatasetSliceStart: (value: string | null) => void; - setDatasetSliceEnd: (value: string | null) => void; setUploadedFile: (file: string | null) => void; setEpochs: (epochs: number) => void; setContextLength: (length: number) => void; diff --git a/studio/tests/test_url_download_benchmark.py b/studio/tests/test_url_download_benchmark.py deleted file mode 100644 index e29a8e05b6..0000000000 --- a/studio/tests/test_url_download_benchmark.py +++ /dev/null @@ -1,54 +0,0 @@ -""" -Benchmark: fsspec URL image download throughput at different dataset sizes. -Dataset: google-research-datasets/conceptual_captions (subset: labeled) - -Tests sizes: 100, 200, 300, 500, 1000, 1500, 2000 -Reports: time, success/fail rate, throughput (images/sec) -""" -from datasets import load_dataset, Dataset -from PIL import Image as PILImage -from io import BytesIO -from itertools import islice -import fsspec -import time - -DATASET = "google-research-datasets/conceptual_captions" -SUBSET = "labeled" -SPLIT = "train" -SIZES = [100, 200, 300, 500, 1000, 1500, 2000] - -# Load the max we need in one go -max_size = max(SIZES) -print(f"Loading {max_size} samples from {DATASET} (streaming)...") -ds = load_dataset(DATASET, name=SUBSET, split=SPLIT, streaming=True) -rows = list(islice(ds, max_size)) -full_dataset = Dataset.from_list(rows) -print(f"Loaded {len(full_dataset)} samples") -print(f"Columns: {full_dataset.column_names}") -print() - -print(f"{'Size':>6} | {'Time':>8} | {'OK':>6} | {'Fail':>6} | {'Fail%':>6} | {'img/s':>7}") -print("-" * 55) - -for size in SIZES: - dataset = full_dataset.select(range(size)) - success, fail = 0, 0 - t0 = time.time() - - for sample in dataset: - url = sample["image_url"] - try: - with fsspec.open(url, "rb", expand=True) as f: - img = PILImage.open(BytesIO(f.read())).convert("RGB") - success += 1 - except Exception: - fail += 1 - - elapsed = time.time() - t0 - fail_pct = (fail / size) * 100 - throughput = success / elapsed if elapsed > 0 else 0 - - print(f"{size:>6} | {elapsed:>7.1f}s | {success:>6} | {fail:>6} | {fail_pct:>5.1f}% | {throughput:>6.1f}/s") - -print() -print("Done.") diff --git a/studio/tests/test_url_image_loading.py b/studio/tests/test_url_image_loading.py deleted file mode 100644 index 4899bab8e3..0000000000 --- a/studio/tests/test_url_image_loading.py +++ /dev/null @@ -1,132 +0,0 @@ -""" -Reproduce: VLM URL image loading with HF datasets. -Tests cast_column(Image()) vs manual download approaches. -Dataset: google-research-datasets/conceptual_captions (subset: labeled) -""" -from datasets import load_dataset, Image as datasets_Image, Dataset -from PIL import Image as PILImage -from io import BytesIO -from itertools import islice -import time - -DATASET = "google-research-datasets/conceptual_captions" -SUBSET = "labeled" -SPLIT = "train" -N_SAMPLES = 20 # small slice for testing - -print("=" * 60) -print("Loading dataset (streaming, first N samples)...") -print("=" * 60) -ds = load_dataset(DATASET, name=SUBSET, split=SPLIT, streaming=True) -rows = list(islice(ds, N_SAMPLES)) -dataset = Dataset.from_list(rows) - -print(f"Loaded {len(dataset)} samples") -print(f"Columns: {dataset.column_names}") -print(f"First image_url: {dataset[0]['image_url'][:100]}...") -print() - -# ─── Test 1: cast_column(Image()) — what we tried ─── -print("=" * 60) -print("TEST 1: cast_column(Image()) approach") -print("=" * 60) -try: - ds_cast = dataset.cast_column("image_url", datasets_Image()) - success, fail = 0, 0 - t0 = time.time() - for i, sample in enumerate(ds_cast): - try: - img = sample["image_url"] - if img is not None: - print(f" [{i}] OK — {img.size} {img.mode}") - success += 1 - else: - print(f" [{i}] None returned") - fail += 1 - except Exception as e: - print(f" [{i}] FAILED: {type(e).__name__}: {str(e)[:80]}") - fail += 1 - elapsed = time.time() - t0 - print(f"\nResult: {success} ok, {fail} failed, {elapsed:.1f}s") -except Exception as e: - print(f"CRASHED during iteration: {type(e).__name__}: {str(e)[:120]}") -print() - -# ─── Test 2: Manual download with requests.Session ─── -print("=" * 60) -print("TEST 2: requests.Session() approach") -print("=" * 60) -try: - import requests - session = requests.Session() - success, fail = 0, 0 - t0 = time.time() - for i, sample in enumerate(dataset): - url = sample["image_url"] - try: - resp = session.get(url, timeout=10) - resp.raise_for_status() - img = PILImage.open(BytesIO(resp.content)).convert("RGB") - print(f" [{i}] OK — {img.size} {img.mode}") - success += 1 - except Exception as e: - print(f" [{i}] FAILED: {type(e).__name__}: {str(e)[:80]}") - fail += 1 - elapsed = time.time() - t0 - print(f"\nResult: {success} ok, {fail} failed, {elapsed:.1f}s") -except Exception as e: - print(f"CRASHED: {type(e).__name__}: {str(e)[:120]}") -print() - -# ─── Test 3: urllib (stdlib) ─── -print("=" * 60) -print("TEST 3: urllib approach (stdlib)") -print("=" * 60) -try: - from urllib.request import urlopen, Request - success, fail = 0, 0 - t0 = time.time() - for i, sample in enumerate(dataset): - url = sample["image_url"] - try: - req = Request(url, headers={"User-Agent": "Mozilla/5.0"}) - with urlopen(req, timeout=10) as resp: - img = PILImage.open(BytesIO(resp.read())).convert("RGB") - print(f" [{i}] OK — {img.size} {img.mode}") - success += 1 - except Exception as e: - print(f" [{i}] FAILED: {type(e).__name__}: {str(e)[:80]}") - fail += 1 - elapsed = time.time() - t0 - print(f"\nResult: {success} ok, {fail} failed, {elapsed:.1f}s") -except Exception as e: - print(f"CRASHED: {type(e).__name__}: {str(e)[:120]}") -print() - -# ─── Test 4: fsspec directly with expand=True ─── -print("=" * 60) -print("TEST 4: fsspec.open() with expand=True") -print("=" * 60) -try: - import fsspec - success, fail = 0, 0 - t0 = time.time() - for i, sample in enumerate(dataset): - url = sample["image_url"] - try: - with fsspec.open(url, "rb", expand=True) as f: - img = PILImage.open(BytesIO(f.read())).convert("RGB") - print(f" [{i}] OK — {img.size} {img.mode}") - success += 1 - except Exception as e: - print(f" [{i}] FAILED: {type(e).__name__}: {str(e)[:80]}") - fail += 1 - elapsed = time.time() - t0 - print(f"\nResult: {success} ok, {fail} failed, {elapsed:.1f}s") -except Exception as e: - print(f"CRASHED: {type(e).__name__}: {str(e)[:120]}") - -print() -print("=" * 60) -print("DONE — compare success rates and timing above") -print("=" * 60) diff --git a/studio/tests/test_url_parallel_benchmark.py b/studio/tests/test_url_parallel_benchmark.py deleted file mode 100644 index a0d160136c..0000000000 --- a/studio/tests/test_url_parallel_benchmark.py +++ /dev/null @@ -1,79 +0,0 @@ -""" -Benchmark: parallel fsspec URL image downloads with ThreadPoolExecutor. -Tests different worker counts to find optimal parallelism. -Dataset: google-research-datasets/conceptual_captions (subset: labeled) -""" -from datasets import load_dataset, Dataset -from PIL import Image as PILImage -from io import BytesIO -from itertools import islice -from concurrent.futures import ThreadPoolExecutor, as_completed -import fsspec -import time -import os - -DATASET = "google-research-datasets/conceptual_captions" -SUBSET = "labeled" -SPLIT = "train" -N_SAMPLES = 500 - -# safe_num_proc formula from studio/backend/utils/hardware/hardware.py -cpu_count = os.cpu_count() -safe_workers = max(1, cpu_count // 3) -print(f"CPU count: {cpu_count}, safe_num_proc: {safe_workers}") - -WORKER_COUNTS = [1, 4, 8, 16, 32, safe_workers] -# Deduplicate and sort -WORKER_COUNTS = sorted(set(WORKER_COUNTS)) - -print(f"Loading {N_SAMPLES} samples from {DATASET} (streaming)...") -ds = load_dataset(DATASET, name=SUBSET, split=SPLIT, streaming=True) -rows = list(islice(ds, N_SAMPLES)) -dataset = Dataset.from_list(rows) -urls = [row["image_url"] for row in dataset] -print(f"Loaded {len(urls)} URLs") -print() - - -def download_single(url): - """Download a single image URL using fsspec. Returns PIL image or raises.""" - with fsspec.open(url, "rb", expand=True) as f: - img = PILImage.open(BytesIO(f.read())).convert("RGB") - return img - - -print(f"{'Workers':>8} | {'Time':>8} | {'OK':>6} | {'Fail':>6} | {'Fail%':>6} | {'img/s':>7} | {'Speedup':>8}") -print("-" * 70) - -baseline_throughput = None - -for n_workers in WORKER_COUNTS: - success, fail = 0, 0 - t0 = time.time() - - with ThreadPoolExecutor(max_workers=n_workers) as pool: - futures = {pool.submit(download_single, url): url for url in urls} - for future in as_completed(futures): - try: - img = future.result(timeout=30) - success += 1 - except Exception: - fail += 1 - - elapsed = time.time() - t0 - fail_pct = (fail / N_SAMPLES) * 100 - throughput = success / elapsed if elapsed > 0 else 0 - - if baseline_throughput is None: - baseline_throughput = throughput - speedup = throughput / baseline_throughput if baseline_throughput > 0 else 0 - - label = f"{n_workers}" - if n_workers == safe_workers: - label += "*" # mark the safe_num_proc value - - print(f"{label:>8} | {elapsed:>7.1f}s | {success:>6} | {fail:>6} | {fail_pct:>5.1f}% | {throughput:>6.1f}/s | {speedup:>7.1f}x") - -print() -print("* = safe_num_proc value") -print("Done.")