feat: add support for serialized previews in dataset API and improve training initialization logging
This commit is contained in:
parent
b0062535a7
commit
d58fa17c81
2 changed files with 53 additions and 11 deletions
|
|
@ -1,6 +1,8 @@
|
|||
"""
|
||||
Datasets API routes
|
||||
"""
|
||||
import base64
|
||||
import io
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from fastapi import APIRouter, HTTPException
|
||||
|
|
@ -30,6 +32,42 @@ if not logger.handlers:
|
|||
from models.datasets import CheckFormatRequest, CheckFormatResponse
|
||||
|
||||
|
||||
def _serialize_preview_value(value):
|
||||
"""make it json safe for client preview ⊂(◉‿◉)つ"""
|
||||
if value is None or isinstance(value, (str, int, float, bool)):
|
||||
return value
|
||||
|
||||
try:
|
||||
from PIL.Image import Image as PILImage
|
||||
if isinstance(value, PILImage):
|
||||
buffer = io.BytesIO()
|
||||
value.convert("RGB").save(buffer, format="JPEG", quality=85)
|
||||
return {
|
||||
"type": "image",
|
||||
"mime": "image/jpeg",
|
||||
"width": value.width,
|
||||
"height": value.height,
|
||||
"data": base64.b64encode(buffer.getvalue()).decode("ascii"),
|
||||
}
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if isinstance(value, dict):
|
||||
return {str(key): _serialize_preview_value(item) for key, item in value.items()}
|
||||
|
||||
if isinstance(value, (list, tuple)):
|
||||
return [_serialize_preview_value(item) for item in value]
|
||||
|
||||
return str(value)
|
||||
|
||||
|
||||
def _serialize_preview_rows(rows):
|
||||
return [
|
||||
{str(key): _serialize_preview_value(value) for key, value in dict(row).items()}
|
||||
for row in rows
|
||||
]
|
||||
|
||||
|
||||
# --- Endpoints ---
|
||||
|
||||
@router.post("/check-format", response_model=CheckFormatResponse)
|
||||
|
|
@ -92,15 +130,15 @@ async def check_format(request: CheckFormatRequest):
|
|||
custom_format_mapping=result.get("suggested_mapping"),
|
||||
)
|
||||
processed = format_result["dataset"]
|
||||
preview_samples = [dict(row) for row in processed]
|
||||
preview_samples = _serialize_preview_rows(processed)
|
||||
except Exception as e:
|
||||
logger.warning(f"Processed preview generation failed (non-fatal): {e}")
|
||||
# Fall back to raw samples so frontend still has something
|
||||
preview_samples = [dict(row) for row in preview_slice]
|
||||
preview_samples = _serialize_preview_rows(preview_slice)
|
||||
else:
|
||||
# Format detection failed — return raw samples so user can
|
||||
# see actual data and map columns in the frontend
|
||||
preview_samples = [dict(row) for row in preview_slice]
|
||||
preview_samples = _serialize_preview_rows(preview_slice)
|
||||
|
||||
return CheckFormatResponse(
|
||||
requires_manual_mapping=result["requires_manual_mapping"],
|
||||
|
|
|
|||
|
|
@ -180,14 +180,19 @@ async def start_training(
|
|||
except Exception as e:
|
||||
logger.error(f"Error updating progress: {e}")
|
||||
|
||||
# Consume the generator - this actually runs the training
|
||||
update_count = 0
|
||||
for _update_tuple in backend.start_training(**training_kwargs):
|
||||
update_count += 1
|
||||
if update_count % 10 == 0:
|
||||
logger.info(f"Training progress update #{update_count}")
|
||||
# start_training returns bool (not generator)
|
||||
run_result = backend.start_training(**training_kwargs)
|
||||
logger.info(
|
||||
"Training job %s backend.start_training returned type=%s value=%r",
|
||||
job_id,
|
||||
type(run_result).__name__,
|
||||
run_result,
|
||||
)
|
||||
if not run_result:
|
||||
progress_error = backend.trainer.training_progress.error
|
||||
raise RuntimeError(progress_error or "Training failed to start")
|
||||
|
||||
logger.info(f"Training job {job_id} completed successfully")
|
||||
logger.info(f"Training job {job_id} started successfully")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Training error in job {job_id}: {e}", exc_info=True)
|
||||
|
|
@ -653,4 +658,3 @@ async def stream_training_progress(
|
|||
"X-Accel-Buffering": "no",
|
||||
}
|
||||
)
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue