ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #311 from unslothai/revert-310-feature/index-range-dataset-slicing

Browse source 
Revert "Add index range dataset slicing to Studio training page"
This commit is contained in:
Roland Tannous 2026-03-05 03:23:31 +04:00 • committed by GitHub
commit 505487f66a
13 changed files with 55 additions and 503 deletions
Download patch file Download diff file Expand all files Collapse all files

24
studio/backend/core/training/trainer.py
View file

@ -353,9 +353,7 @@ class UnslothTrainer:
subset: str = None,
train_split: str = "train",
eval_split: str = None,
eval_steps: float = 0.00,
dataset_slice_start: int = None,
dataset_slice_end: int = None) -> Optional[tuple]:
eval_steps: float = 0.00) -> Optional[tuple]:
"""
Load and prepare dataset for training.
@ -447,18 +445,6 @@ class UnslothTrainer:
if dataset is None:
raise ValueError("No dataset provided")
# Apply index range slicing if requested (inclusive on both ends)
if dataset_slice_start is not None or dataset_slice_end is not None:
total_rows = len(dataset)
start = dataset_slice_start if dataset_slice_start is not None else 0
end = dataset_slice_end if dataset_slice_end is not None else total_rows - 1
# Clamp to valid range
start = max(0, min(start, total_rows - 1))
end = max(start, min(end, total_rows - 1))
dataset = dataset.select(range(start, end + 1))
print(f"Sliced dataset to rows [{start}, {end}]: {len(dataset)} of {total_rows} rows\n")
self._update_progress(status_message=f"Sliced dataset to {len(dataset)} rows (indices {start}-{end})")
# Check if stopped before applying template
if self.should_stop:
print("Stopped before applying chat template\n")
@ -482,14 +468,6 @@ class UnslothTrainer:
print("Stopped during dataset formatting\n")
return None
# Abort if dataset formatting/conversion failed
if not dataset_info.get("success", True):
errors = dataset_info.get("errors", [])
error_msg = "; ".join(errors) if errors else "Dataset formatting failed"
logger.error(f"Dataset conversion failed: {error_msg}")
self._update_progress(error=error_msg)
return None
self._update_progress(status_message=f"Dataset formatted and ready for training")
print(f"Dataset formatted successfully\n")

6
studio/backend/core/training/training.py
View file

@ -116,9 +116,7 @@ class TrainingBackend:
train_split: str = "train",
eval_split: str = None,
eval_steps: float = 0.00,
is_dataset_multimodal: bool = False,
dataset_slice_start: int = None,
dataset_slice_end: int = None) -> bool:
is_dataset_multimodal: bool = False) -> bool:
"""
Start training.
@ -226,8 +224,6 @@ class TrainingBackend:
train_split=train_split,
eval_split=eval_split,
eval_steps=eval_steps,
dataset_slice_start=dataset_slice_start,
dataset_slice_end=dataset_slice_end,
)
# Unpack: load_and_format_dataset returns (dataset, eval_dataset)

2
studio/backend/models/training.py
View file

@ -22,8 +22,6 @@ class TrainingStartRequest(BaseModel):
train_split: Optional[str] = Field("train", description="Training split name")
eval_split: Optional[str] = Field(None, description="Eval split name. None = auto-detect")
eval_steps: float = Field(0.00, description="Fraction of total steps between evals (0-1)")
dataset_slice_start: Optional[int] = Field(None, description="Inclusive start row index for dataset slicing")
dataset_slice_end: Optional[int] = Field(None, description="Inclusive end row index for dataset slicing")
@model_validator(mode="before")
@classmethod

2
studio/backend/routes/training.py
View file

@ -149,8 +149,6 @@ async def start_training(
"train_split": request.train_split,
"eval_split": request.eval_split,
"eval_steps": request.eval_steps,
"dataset_slice_start": request.dataset_slice_start,
"dataset_slice_end": request.dataset_slice_end,
"custom_format_mapping": request.custom_format_mapping,
"num_epochs": request.num_epochs,
"learning_rate": request.learning_rate,

64
studio/backend/utils/datasets/format_conversion.py
View file

@ -281,17 +281,12 @@ def convert_to_vlm_format(
def _convert_single_sample(sample):
"""Convert a single sample to VLM format."""
# Get image (might be PIL Image, local path, or URL)
# Get image (might be PIL Image or path)
image_data = sample[image_column]
# Handle image paths
if isinstance(image_data, str):
if image_data.startswith(("http://", "https://")):
import fsspec
from io import BytesIO
with fsspec.open(image_data, "rb", expand=True) as f:
image_data = Image.open(BytesIO(f.read())).convert("RGB")
else:
image_data = Image.open(image_data).convert("RGB")
image_data = Image.open(image_data).convert("RGB")
# Get text
text_data = sample[text_column]
@ -322,56 +317,11 @@ def convert_to_vlm_format(
# Return dict with messages
return {"messages": messages}
# Convert samples, skipping any with broken/unreachable images.
# For URL-based datasets, check the first PROBE_SIZE samples early to
# fail fast if too many images are broken, before downloading millions.
PROBE_SIZE = 5000
MAX_FAIL_RATE = 0.3
# Use list comprehension and return the LIST directly
print(f"🔄 Converting {len(dataset)} samples to VLM format...")
converted_list = [_convert_single_sample(sample) for sample in dataset]
total = len(dataset)
has_urls = isinstance(next(iter(dataset))[image_column], str)
probe_needed = has_urls and total > PROBE_SIZE
from tqdm import tqdm
print(f"🔄 Converting {total} samples to VLM format...")
converted_list = []
failed_count = 0
pbar = tqdm(dataset, total=total, desc="Converting VLM samples", unit="sample")
for i, sample in enumerate(pbar):
try:
converted_list.append(_convert_single_sample(sample))
except Exception as e:
failed_count += 1
pbar.set_postfix(ok=len(converted_list), failed=failed_count, refresh=False)
# Early exit check after probing the first batch
if probe_needed and (i + 1) == PROBE_SIZE:
fail_rate = failed_count / PROBE_SIZE
if fail_rate >= MAX_FAIL_RATE:
pbar.close()
raise ValueError(
f"{fail_rate:.0%} of the first {PROBE_SIZE} images failed to download "
f"({failed_count}/{PROBE_SIZE}). "
"This dataset has too many broken or unreachable image URLs. "
"Consider using a dataset with embedded images instead."
)
print(f"✅ Probe passed: {failed_count}/{PROBE_SIZE} ({fail_rate:.0%}) failures in first batch, continuing...")
pbar.close()
if failed_count > 0:
fail_rate = failed_count / total
print(f"⚠️ Skipped {failed_count}/{total} ({fail_rate:.0%}) samples with broken/unreachable images")
if len(converted_list) == 0:
raise ValueError(
f"All {total} samples failed during VLM conversion — no usable images found. "
"This dataset may contain only image URLs that are no longer accessible."
)
print(f"✅ Converted {len(converted_list)}/{total} samples")
print(f"✅ Converted {len(converted_list)} samples")
# Return list, NOT Dataset
return converted_list

166
studio/frontend/src/features/studio/sections/dataset-section.tsx
View file

@ -13,7 +13,6 @@ import {
ComboboxItem,
ComboboxList,
} from "@/components/ui/combobox";
import { Input } from "@/components/ui/input";
import { InputGroupAddon } from "@/components/ui/input-group";
import {
Select,
@ -76,10 +75,6 @@ export function DatasetSection() {
setDatasetEvalSplit,
hfToken,
modelType,
datasetSliceStart,
setDatasetSliceStart,
datasetSliceEnd,
setDatasetSliceEnd,
} = useTrainingConfigStore(
useShallow((s) => ({
dataset: s.dataset,
@ -94,10 +89,6 @@ export function DatasetSection() {
setDatasetEvalSplit: s.setDatasetEvalSplit,
hfToken: s.hfToken,
modelType: s.modelType,
datasetSliceStart: s.datasetSliceStart,
setDatasetSliceStart: s.setDatasetSliceStart,
datasetSliceEnd: s.datasetSliceEnd,
setDatasetSliceEnd: s.setDatasetSliceEnd,
})),
);
@ -302,118 +293,51 @@ export function DatasetSection() {
Advanced
</CollapsibleTrigger>
<CollapsibleContent className="mt-3">
<div className="flex flex-col gap-4">
<div className="flex flex-col gap-2">
<span className="flex items-center gap-1.5 text-xs font-medium text-muted-foreground">
Target Format
<Tooltip>
<TooltipTrigger asChild={true}>
<button
type="button"
className="text-foreground/70 hover:text-foreground"
>
<HugeiconsIcon
icon={InformationCircleIcon}
className="size-3"
/>
</button>
</TooltipTrigger>
<TooltipContent>
Format of your training data. Auto-detect works for most
datasets.{" "}
<a
href="https://unsloth.ai/docs/get-started/fine-tuning-llms-guide/datasets-guide"
target="_blank"
rel="noopener noreferrer"
className="text-primary underline"
>
Read more
</a>
</TooltipContent>
</Tooltip>
</span>
<Select
value={datasetFormat}
onValueChange={(v) =>
setDatasetFormat(v as typeof datasetFormat)
}
>
<SelectTrigger className="w-full">
<SelectValue />
</SelectTrigger>
<SelectContent>
<SelectItem value="auto">Auto</SelectItem>
<SelectItem value="alpaca">Alpaca</SelectItem>
<SelectItem value="chatml">ChatML</SelectItem>
<SelectItem value="sharegpt">ShareGPT</SelectItem>
</SelectContent>
</Select>
</div>
<div className="grid grid-cols-2 gap-3">
<div className="flex flex-col gap-1.5">
<span className="flex items-center gap-1.5 text-xs font-medium text-muted-foreground">
Train Split Start
<Tooltip>
<TooltipTrigger asChild={true}>
<button
type="button"
className="text-foreground/70 hover:text-foreground"
>
<HugeiconsIcon
icon={InformationCircleIcon}
className="size-3"
/>
</button>
</TooltipTrigger>
<TooltipContent>
Only train on a subset of your training split by
specifying a start row index (inclusive, 0-based).
Leave empty to start from the first row.
</TooltipContent>
</Tooltip>
</span>
<Input
inputMode="numeric"
placeholder="0"
value={datasetSliceStart ?? ""}
onChange={(e) =>
setDatasetSliceStart(e.target.value || null)
}
/>
</div>
<div className="flex flex-col gap-1.5">
<span className="flex items-center gap-1.5 text-xs font-medium text-muted-foreground">
Train Split End
<Tooltip>
<TooltipTrigger asChild={true}>
<button
type="button"
className="text-foreground/70 hover:text-foreground"
>
<HugeiconsIcon
icon={InformationCircleIcon}
className="size-3"
/>
</button>
</TooltipTrigger>
<TooltipContent>
Last row index to include from the training split
(inclusive, 0-based). For example, set Start to 0 and
End to 99 to train on the first 100 rows. Leave empty
to use all remaining rows.
</TooltipContent>
</Tooltip>
</span>
<Input
inputMode="numeric"
placeholder="End"
value={datasetSliceEnd ?? ""}
onChange={(e) =>
setDatasetSliceEnd(e.target.value || null)
}
/>
</div>
</div>
<div className="flex flex-col gap-2">
<span className="flex items-center gap-1.5 text-xs font-medium text-muted-foreground">
Target Format
<Tooltip>
<TooltipTrigger asChild={true}>
<button
type="button"
className="text-foreground/70 hover:text-foreground"
>
<HugeiconsIcon
icon={InformationCircleIcon}
className="size-3"
/>
</button>
</TooltipTrigger>
<TooltipContent>
Format of your training data. Auto-detect works for most
datasets.{" "}
<a
href="https://unsloth.ai/docs/get-started/fine-tuning-llms-guide/datasets-guide"
target="_blank"
rel="noopener noreferrer"
className="text-primary underline"
>
Read more
</a>
</TooltipContent>
</Tooltip>
</span>
<Select
value={datasetFormat}
onValueChange={(v) =>
setDatasetFormat(v as typeof datasetFormat)
}
>
<SelectTrigger className="w-full">
<SelectValue />
</SelectTrigger>
<SelectContent>
<SelectItem value="auto">Auto</SelectItem>
<SelectItem value="alpaca">Alpaca</SelectItem>
<SelectItem value="chatml">ChatML</SelectItem>
<SelectItem value="sharegpt">ShareGPT</SelectItem>
</SelectContent>
</Select>
</div>
</CollapsibleContent>
</Collapsible>

11
studio/frontend/src/features/training/api/mappers.ts
View file

@ -4,15 +4,6 @@ import type { TrainingStartRequest } from "../types/api";
const BACKEND_LORA_TYPE = "LoRA/QLoRA";
const BACKEND_FULL_TYPE = "Full Finetuning";
function parseSliceValue(value: string | null): number | null {
if (value == null) return null;
const trimmed = value.trim();
if (!trimmed) return null;
const num = Number(trimmed);
if (!Number.isFinite(num) || !Number.isInteger(num)) return null;
return num;
}
export function toBackendTrainingType(trainingMethod: string): string {
return trainingMethod === "full" ? BACKEND_FULL_TYPE : BACKEND_LORA_TYPE;
}
@ -36,8 +27,6 @@ export function buildTrainingStartPayload(
subset: hfDataset ? config.datasetSubset : null,
train_split: hfDataset ? config.datasetSplit : null,
eval_split: hfDataset ? config.datasetEvalSplit : null,
dataset_slice_start: parseSliceValue(config.datasetSliceStart),
dataset_slice_end: parseSliceValue(config.datasetSliceEnd),
local_datasets: [],
format_type: config.datasetFormat,
custom_format_mapping: customFormatMapping,

12
studio/frontend/src/features/training/stores/training-config-store.ts
View file

@ -28,8 +28,6 @@ const initialState: TrainingConfigState = {
datasetSplit: null,
datasetEvalSplit: null,
datasetManualMapping: emptyManualMapping(),
datasetSliceStart: null,
datasetSliceEnd: null,
uploadedFile: null,
isCheckingVision: false,
isVisionModel: false,
@ -257,8 +255,6 @@ export const useTrainingConfigStore = create<TrainingConfigStore>()(
datasetSplit: null,
datasetEvalSplit: null,
datasetManualMapping: emptyManualMapping(),
datasetSliceStart: null,
datasetSliceEnd: null,
isDatasetMultimodal: null,
isCheckingDataset: false,
});
@ -315,8 +311,6 @@ export const useTrainingConfigStore = create<TrainingConfigStore>()(
},
setDatasetManualMapping: (datasetManualMapping) =>
set({ datasetManualMapping }),
setDatasetSliceStart: (datasetSliceStart) => set({ datasetSliceStart }),
setDatasetSliceEnd: (datasetSliceEnd) => set({ datasetSliceEnd }),
setUploadedFile: (uploadedFile) => set({ uploadedFile }),
setEpochs: (epochs) => set({ epochs }),
setContextLength: (contextLength) => set({ contextLength }),
@ -374,7 +368,7 @@ export const useTrainingConfigStore = create<TrainingConfigStore>()(
},
{
name: "unsloth_training_config_v1",
version: 7,
version: 6,
migrate: (persisted, version) => {
const s = persisted as Record<string, unknown>;
if (version < 2 && s.datasetSubset == null && s.datasetConfig != null) {
@ -393,10 +387,6 @@ export const useTrainingConfigStore = create<TrainingConfigStore>()(
if (version < 6 && s.datasetEvalSplit == null) {
s.datasetEvalSplit = null;
}
if (version < 7) {
s.datasetSliceStart ??= null;
s.datasetSliceEnd ??= null;
}
return s as unknown as TrainingConfigStore;
},
partialize: partializePersistedState,

2
studio/frontend/src/features/training/types/api.ts
View file

@ -8,8 +8,6 @@ export interface TrainingStartRequest {
subset: string | null;
train_split: string | null;
eval_split: string | null;
dataset_slice_start: number | null;
dataset_slice_end: number | null;
local_datasets: string[];
format_type: string;
custom_format_mapping?: Record<string, string> | null;

4
studio/frontend/src/features/training/types/config.ts
View file

@ -26,8 +26,6 @@ export interface TrainingConfigState {
datasetSplit: string | null;
datasetEvalSplit: string | null;
datasetManualMapping: DatasetManualMapping;
datasetSliceStart: string | null;
datasetSliceEnd: string | null;
uploadedFile: string | null;
epochs: number;
contextLength: number;
@ -86,8 +84,6 @@ export interface TrainingConfigActions {
setDatasetSplit: (split: string | null) => void;
setDatasetEvalSplit: (split: string | null) => void;
setDatasetManualMapping: (mapping: DatasetManualMapping) => void;
setDatasetSliceStart: (value: string | null) => void;
setDatasetSliceEnd: (value: string | null) => void;
setUploadedFile: (file: string | null) => void;
setEpochs: (epochs: number) => void;
setContextLength: (length: number) => void;

54
studio/tests/test_url_download_benchmark.py
View file

@ -1,54 +0,0 @@
"""
Benchmark: fsspec URL image download throughput at different dataset sizes.
Dataset: google-research-datasets/conceptual_captions (subset: labeled)
Tests sizes: 100, 200, 300, 500, 1000, 1500, 2000
Reports: time, success/fail rate, throughput (images/sec)
"""
from datasets import load_dataset, Dataset
from PIL import Image as PILImage
from io import BytesIO
from itertools import islice
import fsspec
import time
DATASET = "google-research-datasets/conceptual_captions"
SUBSET = "labeled"
SPLIT = "train"
SIZES = [100, 200, 300, 500, 1000, 1500, 2000]
# Load the max we need in one go
max_size = max(SIZES)
print(f"Loading {max_size} samples from {DATASET} (streaming)...")
ds = load_dataset(DATASET, name=SUBSET, split=SPLIT, streaming=True)
rows = list(islice(ds, max_size))
full_dataset = Dataset.from_list(rows)
print(f"Loaded {len(full_dataset)} samples")
print(f"Columns: {full_dataset.column_names}")
print()
print(f"{'Size':>6} | {'Time':>8} | {'OK':>6} | {'Fail':>6} | {'Fail%':>6} | {'img/s':>7}")
print("-" * 55)
for size in SIZES:
dataset = full_dataset.select(range(size))
success, fail = 0, 0
t0 = time.time()
for sample in dataset:
url = sample["image_url"]
try:
with fsspec.open(url, "rb", expand=True) as f:
img = PILImage.open(BytesIO(f.read())).convert("RGB")
success += 1
except Exception:
fail += 1
elapsed = time.time() - t0
fail_pct = (fail / size) * 100
throughput = success / elapsed if elapsed > 0 else 0
print(f"{size:>6} | {elapsed:>7.1f}s | {success:>6} | {fail:>6} | {fail_pct:>5.1f}% | {throughput:>6.1f}/s")
print()
print("Done.")

132
studio/tests/test_url_image_loading.py
View file

@ -1,132 +0,0 @@
"""
Reproduce: VLM URL image loading with HF datasets.
Tests cast_column(Image()) vs manual download approaches.
Dataset: google-research-datasets/conceptual_captions (subset: labeled)
"""
from datasets import load_dataset, Image as datasets_Image, Dataset
from PIL import Image as PILImage
from io import BytesIO
from itertools import islice
import time
DATASET = "google-research-datasets/conceptual_captions"
SUBSET = "labeled"
SPLIT = "train"
N_SAMPLES = 20 # small slice for testing
print("=" * 60)
print("Loading dataset (streaming, first N samples)...")
print("=" * 60)
ds = load_dataset(DATASET, name=SUBSET, split=SPLIT, streaming=True)
rows = list(islice(ds, N_SAMPLES))
dataset = Dataset.from_list(rows)
print(f"Loaded {len(dataset)} samples")
print(f"Columns: {dataset.column_names}")
print(f"First image_url: {dataset[0]['image_url'][:100]}...")
print()
# ─── Test 1: cast_column(Image()) — what we tried ───
print("=" * 60)
print("TEST 1: cast_column(Image()) approach")
print("=" * 60)
try:
ds_cast = dataset.cast_column("image_url", datasets_Image())
success, fail = 0, 0
t0 = time.time()
for i, sample in enumerate(ds_cast):
try:
img = sample["image_url"]
if img is not None:
print(f" [{i}] OK — {img.size} {img.mode}")
success += 1
else:
print(f" [{i}] None returned")
fail += 1
except Exception as e:
print(f" [{i}] FAILED: {type(e).__name__}: {str(e)[:80]}")
fail += 1
elapsed = time.time() - t0
print(f"\nResult: {success} ok, {fail} failed, {elapsed:.1f}s")
except Exception as e:
print(f"CRASHED during iteration: {type(e).__name__}: {str(e)[:120]}")
print()
# ─── Test 2: Manual download with requests.Session ───
print("=" * 60)
print("TEST 2: requests.Session() approach")
print("=" * 60)
try:
import requests
session = requests.Session()
success, fail = 0, 0
t0 = time.time()
for i, sample in enumerate(dataset):
url = sample["image_url"]
try:
resp = session.get(url, timeout=10)
resp.raise_for_status()
img = PILImage.open(BytesIO(resp.content)).convert("RGB")
print(f" [{i}] OK — {img.size} {img.mode}")
success += 1
except Exception as e:
print(f" [{i}] FAILED: {type(e).__name__}: {str(e)[:80]}")
fail += 1
elapsed = time.time() - t0
print(f"\nResult: {success} ok, {fail} failed, {elapsed:.1f}s")
except Exception as e:
print(f"CRASHED: {type(e).__name__}: {str(e)[:120]}")
print()
# ─── Test 3: urllib (stdlib) ───
print("=" * 60)
print("TEST 3: urllib approach (stdlib)")
print("=" * 60)
try:
from urllib.request import urlopen, Request
success, fail = 0, 0
t0 = time.time()
for i, sample in enumerate(dataset):
url = sample["image_url"]
try:
req = Request(url, headers={"User-Agent": "Mozilla/5.0"})
with urlopen(req, timeout=10) as resp:
img = PILImage.open(BytesIO(resp.read())).convert("RGB")
print(f" [{i}] OK — {img.size} {img.mode}")
success += 1
except Exception as e:
print(f" [{i}] FAILED: {type(e).__name__}: {str(e)[:80]}")
fail += 1
elapsed = time.time() - t0
print(f"\nResult: {success} ok, {fail} failed, {elapsed:.1f}s")
except Exception as e:
print(f"CRASHED: {type(e).__name__}: {str(e)[:120]}")
print()
# ─── Test 4: fsspec directly with expand=True ───
print("=" * 60)
print("TEST 4: fsspec.open() with expand=True")
print("=" * 60)
try:
import fsspec
success, fail = 0, 0
t0 = time.time()
for i, sample in enumerate(dataset):
url = sample["image_url"]
try:
with fsspec.open(url, "rb", expand=True) as f:
img = PILImage.open(BytesIO(f.read())).convert("RGB")
print(f" [{i}] OK — {img.size} {img.mode}")
success += 1
except Exception as e:
print(f" [{i}] FAILED: {type(e).__name__}: {str(e)[:80]}")
fail += 1
elapsed = time.time() - t0
print(f"\nResult: {success} ok, {fail} failed, {elapsed:.1f}s")
except Exception as e:
print(f"CRASHED: {type(e).__name__}: {str(e)[:120]}")
print()
print("=" * 60)
print("DONE — compare success rates and timing above")
print("=" * 60)

79
studio/tests/test_url_parallel_benchmark.py
View file

@ -1,79 +0,0 @@
"""
Benchmark: parallel fsspec URL image downloads with ThreadPoolExecutor.
Tests different worker counts to find optimal parallelism.
Dataset: google-research-datasets/conceptual_captions (subset: labeled)
"""
from datasets import load_dataset, Dataset
from PIL import Image as PILImage
from io import BytesIO
from itertools import islice
from concurrent.futures import ThreadPoolExecutor, as_completed
import fsspec
import time
import os
DATASET = "google-research-datasets/conceptual_captions"
SUBSET = "labeled"
SPLIT = "train"
N_SAMPLES = 500
# safe_num_proc formula from studio/backend/utils/hardware/hardware.py
cpu_count = os.cpu_count()
safe_workers = max(1, cpu_count // 3)
print(f"CPU count: {cpu_count}, safe_num_proc: {safe_workers}")
WORKER_COUNTS = [1, 4, 8, 16, 32, safe_workers]
# Deduplicate and sort
WORKER_COUNTS = sorted(set(WORKER_COUNTS))
print(f"Loading {N_SAMPLES} samples from {DATASET} (streaming)...")
ds = load_dataset(DATASET, name=SUBSET, split=SPLIT, streaming=True)
rows = list(islice(ds, N_SAMPLES))
dataset = Dataset.from_list(rows)
urls = [row["image_url"] for row in dataset]
print(f"Loaded {len(urls)} URLs")
print()
def download_single(url):
"""Download a single image URL using fsspec. Returns PIL image or raises."""
with fsspec.open(url, "rb", expand=True) as f:
img = PILImage.open(BytesIO(f.read())).convert("RGB")
return img
print(f"{'Workers':>8} | {'Time':>8} | {'OK':>6} | {'Fail':>6} | {'Fail%':>6} | {'img/s':>7} | {'Speedup':>8}")
print("-" * 70)
baseline_throughput = None
for n_workers in WORKER_COUNTS:
success, fail = 0, 0
t0 = time.time()
with ThreadPoolExecutor(max_workers=n_workers) as pool:
futures = {pool.submit(download_single, url): url for url in urls}
for future in as_completed(futures):
try:
img = future.result(timeout=30)
success += 1
except Exception:
fail += 1
elapsed = time.time() - t0
fail_pct = (fail / N_SAMPLES) * 100
throughput = success / elapsed if elapsed > 0 else 0
if baseline_throughput is None:
baseline_throughput = throughput
speedup = throughput / baseline_throughput if baseline_throughput > 0 else 0
label = f"{n_workers}"
if n_workers == safe_workers:
label += "*" # mark the safe_num_proc value
print(f"{label:>8} | {elapsed:>7.1f}s | {success:>6} | {fail:>6} | {fail_pct:>5.1f}% | {throughput:>6.1f}/s | {speedup:>7.1f}x")
print()
print("* = safe_num_proc value")
print("Done.")