fix: prefer tabular files over archives in Tier 1 dataset preview

Tier 1 check-format was picking images.zip over testmini.parquet,
causing wrong columns (image/label) and broken VLM mapping.
Also log first VLM conversion failure instead of swallowing silently.
This commit is contained in:
Roland Tannous 2026-03-09 22:00:20 +00:00
commit c272c4f844
2 changed files with 27 additions and 14 deletions

View file

@ -83,16 +83,12 @@ def _serialize_preview_rows(rows):
# --- Endpoints ---
# Recognized data-file extensions for the single-file fallback approach.
DATA_EXTS = (
'.parquet',
'.json', '.jsonl',
'.csv', '.tsv',
'.txt',
'.arrow',
'.tar', '.tar.gz', '.tgz',
'.gz', '.zst',
'.zip',
)
# Tabular formats are preferred over archives for Tier 1 preview because
# archives (e.g. images.zip) may be loaded as ImageFolder datasets with
# synthetic columns (image/label) that don't match the real dataset schema.
_TABULAR_EXTS = ('.parquet', '.json', '.jsonl', '.csv', '.tsv', '.arrow')
_ARCHIVE_EXTS = ('.tar', '.tar.gz', '.tgz', '.gz', '.zst', '.zip', '.txt')
DATA_EXTS = _TABULAR_EXTS + _ARCHIVE_EXTS
LOCAL_FILE_EXTS = ('.json', '.jsonl', '.csv', '.parquet')
LOCAL_UPLOAD_EXTS = {".csv", ".json", ".jsonl", ".parquet"}
BACKEND_ROOT = Path(__file__).resolve().parents[1]
@ -363,8 +359,20 @@ def check_format(
)
data_files = [f for f in repo_files if any(f.endswith(ext) for ext in DATA_EXTS)]
if data_files:
first_file = data_files[0]
# Prefer tabular formats over archives (e.g. images.zip → ImageFolder
# with synthetic image/label columns that don't match the real schema).
tabular_files = [f for f in data_files if any(f.endswith(ext) for ext in _TABULAR_EXTS)]
candidates = tabular_files or data_files
# When a subset is specified, narrow to files whose name matches
# (e.g. subset="testmini" → prefer "testmini.parquet").
if request.subset and candidates:
subset_matches = [f for f in candidates if request.subset in Path(f).stem]
if subset_matches:
candidates = subset_matches
if candidates:
first_file = candidates[0]
logger.info(f"Tier 1: loading single file {first_file}")
load_kwargs = {
"path": request.dataset_name,

View file

@ -443,8 +443,10 @@ def convert_to_vlm_format(
idx = futures[future]
try:
batch_results[idx] = future.result()
except Exception:
except Exception as e:
failed_count += 1
if failed_count == 1:
print(f"⚠️ First VLM conversion failure: {type(e).__name__}: {e}")
converted_list.extend(r for r in batch_results if r is not None)
@ -463,8 +465,11 @@ def convert_to_vlm_format(
for sample in pbar:
try:
converted_list.append(_convert_single_sample(sample))
except Exception:
except Exception as e:
failed_count += 1
if failed_count == 1:
# Log the first failure to aid debugging
print(f"⚠️ First VLM conversion failure: {type(e).__name__}: {e}")
pbar.set_postfix(ok=len(converted_list), failed=failed_count, refresh=False)
pbar.close()