feat(datasets): check-format to return preview samples

This commit is contained in:
Roland Tannous 2026-02-12 11:21:38 +00:00
commit 4c791bd5aa
4 changed files with 33 additions and 8 deletions

View file

@ -21,3 +21,5 @@ class CheckFormatResponse(BaseModel):
suggested_mapping: Optional[Dict[str, str]] = None
detected_image_column: Optional[str] = None
detected_text_column: Optional[str] = None
preview_samples: Optional[List[Dict]] = None
total_rows: Optional[int] = None

View file

@ -37,12 +37,15 @@ async def check_format(request: CheckFormatRequest):
"""
Check if a dataset requires manual column mapping.
This is a lightweight check that only runs format detection,
not full processing. Use before starting training to determine
if the user needs to manually map columns.
This is a lightweight check that loads only the first 10 rows,
runs format detection, and (if processable) returns processed
preview samples. The full dataset is re-processed at training time.
"""
try:
from datasets import load_dataset
from utils.datasets import format_dataset
PREVIEW_SIZE = 10
logger.info(f"Checking format for dataset: {request.dataset_name}")
@ -69,11 +72,29 @@ async def check_format(request: CheckFormatRequest):
load_kwargs["token"] = request.hf_token
dataset = load_dataset(**load_kwargs)
# Run lightweight format check
result = check_dataset_format(dataset, is_vlm=request.is_vlm)
# Slice to top N rows — all detection and preview runs on this subset
total_rows = len(dataset)
preview_slice = dataset.select(range(min(PREVIEW_SIZE, total_rows)))
# Run lightweight format check on the preview slice
result = check_dataset_format(preview_slice, is_vlm=request.is_vlm)
logger.info(f"Format check result: requires_mapping={result['requires_manual_mapping']}, format={result['detected_format']}")
# If format is processable, generate preview samples via format_dataset
preview_samples = None
if not result["requires_manual_mapping"]:
try:
format_result = format_dataset(
preview_slice,
format_type="auto",
custom_format_mapping=result.get("suggested_mapping"),
)
processed = format_result["dataset"]
preview_samples = [dict(row) for row in processed]
except Exception as e:
logger.warning(f"Preview generation failed (non-fatal): {e}")
return CheckFormatResponse(
requires_manual_mapping=result["requires_manual_mapping"],
detected_format=result["detected_format"],
@ -81,6 +102,8 @@ async def check_format(request: CheckFormatRequest):
suggested_mapping=result.get("suggested_mapping"),
detected_image_column=result.get("detected_image_column"),
detected_text_column=result.get("detected_text_column"),
preview_samples=preview_samples,
total_rows=total_rows,
)
except HTTPException:

View file

@ -14,7 +14,7 @@ if str(backend_dir) not in sys.path:
def run_server(
host: str = "0.0.0.0",
port: int = 8000,
frontend_path: Path = None,
frontend_path: Path = "../frontend/dist",
silent: bool = False,
):
"""
@ -75,7 +75,7 @@ if __name__ == "__main__":
parser.add_argument("--host", default="0.0.0.0", help="Host to bind to")
parser.add_argument("--port", type=int, default=8000, help="Port to bind to")
parser.add_argument(
"--frontend", type=str, default=None, help="Path to frontend build"
"--frontend", type=str, default="../frontend/dist", help="Path to frontend build"
)
parser.add_argument("--silent", action="store_true", help="Suppress output")

View file

@ -11,7 +11,7 @@ function collectTextParts(message: RunMessage): string[] {
.map((c) => c.text);
if ("attachments" in message && (message.attachments?.length ?? 0) > 0) {
for (const att of message.attachments) {
for (const att of message.attachments ?? []) {
for (const part of att.content ?? []) {
if (part.type === "text") {
textParts.push(part.text);