From 4e4fc367b6581f1b28e5f33ac26d0ac11d503b4c Mon Sep 17 00:00:00 2001 From: Roland Tannous Date: Fri, 13 Feb 2026 17:29:39 +0000 Subject: [PATCH] fix: auto-detect multimodal datasets in /check-format without requiring is_vlm flag --- studio/backend/models/datasets.py | 2 ++ studio/backend/routes/datasets.py | 2 ++ studio/backend/utils/datasets/dataset_utils.py | 13 +++++++++++++ 3 files changed, 17 insertions(+) diff --git a/studio/backend/models/datasets.py b/studio/backend/models/datasets.py index d714cd397d..530a9fa30b 100644 --- a/studio/backend/models/datasets.py +++ b/studio/backend/models/datasets.py @@ -18,6 +18,8 @@ class CheckFormatResponse(BaseModel): requires_manual_mapping: bool detected_format: str columns: List[str] + is_multimodal: bool = False + multimodal_columns: Optional[List[str]] = None suggested_mapping: Optional[Dict[str, str]] = None detected_image_column: Optional[str] = None detected_text_column: Optional[str] = None diff --git a/studio/backend/routes/datasets.py b/studio/backend/routes/datasets.py index 72c36307c9..e462bbfe18 100644 --- a/studio/backend/routes/datasets.py +++ b/studio/backend/routes/datasets.py @@ -144,6 +144,8 @@ async def check_format(request: CheckFormatRequest): requires_manual_mapping=result["requires_manual_mapping"], detected_format=result["detected_format"], columns=result["columns"], + is_multimodal=result.get("is_multimodal", False), + multimodal_columns=result.get("multimodal_columns"), suggested_mapping=result.get("suggested_mapping"), detected_image_column=result.get("detected_image_column"), detected_text_column=result.get("detected_text_column"), diff --git a/studio/backend/utils/datasets/dataset_utils.py b/studio/backend/utils/datasets/dataset_utils.py index c16ea3fe63..a75f78d37c 100644 --- a/studio/backend/utils/datasets/dataset_utils.py +++ b/studio/backend/utils/datasets/dataset_utils.py @@ -63,6 +63,11 @@ def check_dataset_format(dataset, is_vlm: bool = False) -> dict: """ columns = list(dataset.column_names) if hasattr(dataset, 'column_names') else list(next(iter(dataset)).keys()) + # Auto-detect multimodal data regardless of is_vlm flag + multimodal_info = detect_multimodal_dataset(dataset) + if multimodal_info["is_multimodal"]: + is_vlm = True # Route to VLM detection automatically + if is_vlm: vlm_structure = detect_vlm_dataset_structure(dataset) requires_mapping = vlm_structure["format"] == "unknown" @@ -74,6 +79,8 @@ def check_dataset_format(dataset, is_vlm: bool = False) -> dict: "suggested_mapping": None, "detected_image_column": vlm_structure.get("image_column"), "detected_text_column": vlm_structure.get("text_column"), + "is_multimodal": multimodal_info["is_multimodal"], + "multimodal_columns": multimodal_info.get("multimodal_columns"), } else: # LLM flow @@ -91,6 +98,8 @@ def check_dataset_format(dataset, is_vlm: bool = False) -> dict: "suggested_mapping": heuristic_mapping, "detected_image_column": None, "detected_text_column": None, + "is_multimodal": False, + "multimodal_columns": None, } else: # Both detection and heuristic failed @@ -101,6 +110,8 @@ def check_dataset_format(dataset, is_vlm: bool = False) -> dict: "suggested_mapping": None, "detected_image_column": None, "detected_text_column": None, + "is_multimodal": False, + "multimodal_columns": None, } # Known format detected @@ -111,6 +122,8 @@ def check_dataset_format(dataset, is_vlm: bool = False) -> dict: "suggested_mapping": None, "detected_image_column": None, "detected_text_column": None, + "is_multimodal": False, + "multimodal_columns": None, } def _apply_user_mapping(dataset, mapping: dict, batch_size: int = 1000):