feat: add ShareGPT+image VLM format support and improve image column detection

- Detect and convert ShareGPT/ChatML conversations with <image> placeholders
- Add file_name/filename as image column keywords
- Detect image paths and URLs by value (string ending in .jpg/.png/etc)

studio/backend/utils/datasets/__init__.py

@@ -31,6 +31,7 @@ from .format_conversion import (
     convert_alpaca_to_chatml,
     convert_to_vlm_format,
     convert_llava_to_vlm_format,
+    convert_sharegpt_with_images_to_vlm_format,
 )
 
 # Chat templates
@@ -81,6 +82,7 @@ __all__ = [
     "convert_alpaca_to_chatml",
     "convert_to_vlm_format",
     "convert_llava_to_vlm_format",
+    "convert_sharegpt_with_images_to_vlm_format",
     # Templates
     "apply_chat_template_to_dataset",
     "get_dataset_info_summary",

studio/backend/utils/datasets/dataset_utils.py

@@ -31,6 +31,7 @@ from .format_conversion import (
     convert_alpaca_to_chatml,
     convert_to_vlm_format,
     convert_llava_to_vlm_format,
+    convert_sharegpt_with_images_to_vlm_format,
 )
 from .chat_templates import (
     apply_chat_template_to_dataset,
@@ -738,6 +739,33 @@ def format_and_template_dataset(
                     "errors": errors,
                 }
 
+        # Handle ShareGPT/ChatML + image column (e.g. ShareGPT4V, LLaVA-style)
+        elif vlm_structure["format"] == "sharegpt_with_images":
+            try:
+                dataset = convert_sharegpt_with_images_to_vlm_format(
+                    dataset,
+                    image_column=vlm_structure["image_column"],
+                    messages_column=vlm_structure["messages_column"],
+                    dataset_name=dataset_name,
+                    progress_callback=progress_callback,
+                )
+                warnings.append("Converted from ShareGPT+image format to standard VLM format")
+            except Exception as e:
+                errors.append(f"Failed to convert ShareGPT+image format: {e}")
+                import traceback
+                traceback.print_exc()
+
+                return {
+                    "dataset": dataset,
+                    "detected_format": "sharegpt_with_images",
+                    "final_format": "vlm_conversion_failed",
+                    "is_vlm": True,
+                    "success": False,
+                    "requires_manual_mapping": True,
+                    "warnings": warnings,
+                    "errors": errors,
+                }
+
         # Handle simple format
         elif vlm_structure["needs_conversion"]:
             if vlm_text_column is None:

studio/backend/utils/datasets/format_conversion.py

@@ -538,6 +538,164 @@ def convert_to_vlm_format(
     return converted_list
 
 
+def convert_sharegpt_with_images_to_vlm_format(
+    dataset,
+    image_column="image",
+    messages_column="conversations",
+    dataset_name=None,
+    progress_callback=None,
+):
+    """
+    Converts ShareGPT/ChatML datasets that have a separate image column and
+    ``<image>`` placeholders inside the conversation text.
+
+    Example input::
+
+        {
+            "image": "sam/images/sa_545504.jpg",
+            "conversations": [
+                {"from": "human", "value": "<image>\\nWhat is this photo about?"},
+                {"from": "gpt",   "value": "The image captures..."}
+            ]
+        }
+
+    Returns a list of dicts in standard VLM messages format (PIL Images inline).
+    """
+    from PIL import Image
+    from tqdm import tqdm
+
+    _IMAGE_EXTS = ('.png', '.jpg', '.jpeg', '.webp', '.gif', '.bmp', '.tiff')
+    _ROLE_MAP = {
+        "human": "user", "user": "user",
+        "gpt": "assistant", "assistant": "assistant",
+        "system": "system",
+    }
+
+    def _notify(msg):
+        if progress_callback:
+            progress_callback(status_message=msg)
+
+    # ── Resolve image loading strategy (same 3-tier as convert_to_vlm_format) ──
+    total = len(dataset)
+    first_image = next(iter(dataset))[image_column]
+
+    _image_lookup = None
+    if (
+        isinstance(first_image, str)
+        and not first_image.startswith(("http://", "https://"))
+        and not os.path.exists(first_image)
+        and dataset_name
+    ):
+        try:
+            from huggingface_hub import HfApi
+            _notify("Resolving image filenames from HF repo...")
+            print(f"🔍 Image column contains bare filenames (e.g. '{first_image}') — building repo lookup...")
+            repo_files = HfApi().list_repo_files(dataset_name, repo_type="dataset")
+            _image_lookup = {
+                os.path.basename(f): f
+                for f in repo_files
+                if any(f.lower().endswith(ext) for ext in _IMAGE_EXTS)
+            }
+            # Also add the full relative paths as keys (for paths like "sam/images/sa_545504.jpg")
+            for f in repo_files:
+                if any(f.lower().endswith(ext) for ext in _IMAGE_EXTS):
+                    _image_lookup[f] = f
+            if first_image in _image_lookup:
+                print(f"✅ Matched {len(_image_lookup)} image files in repo (e.g. '{first_image}' → '{_image_lookup[first_image]}')")
+            else:
+                print(f"⚠️ Built lookup with {len(_image_lookup)} images but '{first_image}' not found — falling back to local open")
+                _image_lookup = None
+        except Exception as e:
+            print(f"⚠️ Failed to build HF repo image lookup: {e}")
+            _image_lookup = None
+
+    def _resolve_image(image_data):
+        """Resolve image data to a PIL Image object."""
+        if hasattr(image_data, 'size') and hasattr(image_data, 'mode'):
+            return image_data  # Already PIL
+        if isinstance(image_data, str):
+            if image_data.startswith(("http://", "https://")):
+                import fsspec
+                from io import BytesIO
+                with fsspec.open(image_data, "rb", expand=True) as f:
+                    return Image.open(BytesIO(f.read())).convert("RGB")
+            elif _image_lookup is not None and image_data in _image_lookup:
+                from huggingface_hub import hf_hub_download
+                local_path = hf_hub_download(
+                    dataset_name, _image_lookup[image_data], repo_type="dataset",
+                )
+                return Image.open(local_path).convert("RGB")
+            else:
+                return Image.open(image_data).convert("RGB")
+        if isinstance(image_data, dict) and ("bytes" in image_data or "path" in image_data):
+            if image_data.get("bytes"):
+                from io import BytesIO
+                return Image.open(BytesIO(image_data["bytes"])).convert("RGB")
+            if image_data.get("path"):
+                return Image.open(image_data["path"]).convert("RGB")
+        raise ValueError(f"Cannot resolve image: {type(image_data)}")
+
+    def _convert_single_sample(sample):
+        """Convert a single ShareGPT+image sample to standard VLM format."""
+        pil_image = _resolve_image(sample[image_column])
+        conversation = sample[messages_column]
+
+        new_messages = []
+        for msg in conversation:
+            role_raw = msg.get("from") or msg.get("role", "user")
+            role = _ROLE_MAP.get(role_raw.lower(), role_raw.lower())
+            text = msg.get("value") or msg.get("content") or ""
+
+            # Split on <image> to interleave text and image content blocks
+            if "<image>" in text:
+                parts = text.split("<image>")
+                content = []
+                for i, part in enumerate(parts):
+                    part = part.strip()
+                    if part:
+                        content.append({"type": "text", "text": part})
+                    if i < len(parts) - 1:
+                        content.append({"type": "image", "image": pil_image})
+                # If <image> was the entire text, content might just be the image
+                if not content:
+                    content.append({"type": "image", "image": pil_image})
+            else:
+                content = [{"type": "text", "text": text}]
+
+            new_messages.append({"role": role, "content": content})
+
+        return {"messages": new_messages}
+
+    # ── Full conversion with progress ──
+    print(f"🔄 Converting {total} samples from ShareGPT+image format...")
+    converted_list = []
+    failed_count = 0
+
+    pbar = tqdm(dataset, total=total, desc="Converting ShareGPT+image", unit="sample")
+    for sample in pbar:
+        try:
+            converted_list.append(_convert_single_sample(sample))
+        except Exception as e:
+            failed_count += 1
+            if failed_count == 1:
+                print(f"⚠️ First conversion failure: {type(e).__name__}: {e}")
+        pbar.set_postfix(ok=len(converted_list), failed=failed_count, refresh=False)
+    pbar.close()
+
+    if failed_count > 0:
+        print(f"⚠️ Skipped {failed_count}/{total} ({failed_count*100//total}%) samples")
+
+    if len(converted_list) == 0:
+        raise ValueError(
+            f"All {total} samples failed during ShareGPT+image conversion — "
+            "no usable samples found."
+        )
+
+    print(f"✅ Converted {len(converted_list)}/{total} samples")
+    _notify(f"Converted {len(converted_list):,}/{total:,} samples successfully")
+    return converted_list
+
+
 def convert_llava_to_vlm_format(dataset):
     """
     Converts Llava format to standard VLM format.

studio/backend/utils/datasets/format_detection.py

@@ -361,6 +361,7 @@ def detect_multimodal_dataset(dataset):
         'image', 'img', 'pixel',
         'jpg', 'jpeg', 'png', 'webp', 'bmp', 'gif', 'tiff', 'svg',
         'photo', 'pic', 'picture', 'visual',
+        'file_name', 'filename',
     ]
 
     # Keywords that indicate audio data
@@ -477,6 +478,17 @@ def _is_image_value(value) -> bool:
     if isinstance(value, (bytes, bytearray)):
         return _has_image_header(value)
 
+    # String that looks like an image file path or URL
+    _IMAGE_EXTS = ('.png', '.jpg', '.jpeg', '.webp', '.gif', '.bmp', '.tiff', '.svg')
+    if isinstance(value, str) and len(value) < 1000:
+        lower = value.strip().lower()
+        # Image URL (http://... ending in image extension)
+        if lower.startswith(("http://", "https://")) and any(lower.split("?")[0].endswith(ext) for ext in _IMAGE_EXTS):
+            return True
+        # Image file path (relative or absolute path ending in image extension)
+        if any(lower.endswith(ext) for ext in _IMAGE_EXTS):
+            return True
+
     return False
 
 
@@ -581,6 +593,46 @@ def detect_vlm_dataset_structure(dataset):
                                 "text_column": None,
                             }
 
+    # Check for ShareGPT/ChatML conversations with <image> placeholder + companion image column
+    # (e.g. Lin-Chen/ShareGPT4V, LLaVA-style datasets)
+    for chat_col in ("conversations", "messages"):
+        if chat_col not in column_names:
+            continue
+        chat_data = sample[chat_col]
+        if not isinstance(chat_data, list) or len(chat_data) == 0:
+            continue
+        first_msg = chat_data[0]
+        if not isinstance(first_msg, dict):
+            continue
+        # Detect ShareGPT (from/value) or ChatML (role/content) keys
+        msg_text = first_msg.get("value") or first_msg.get("content")
+        if not isinstance(msg_text, str):
+            continue
+        # Check for <image> placeholder anywhere in the conversation
+        has_image_placeholder = any(
+            "<image>" in str(m.get("value", "") or m.get("content", ""))
+            for m in chat_data
+            if isinstance(m, dict)
+        )
+        if not has_image_placeholder:
+            continue
+        # Find companion image column
+        image_col = None
+        for col in column_names:
+            if col == chat_col:
+                continue
+            if _keyword_in_column("image", col) or _keyword_in_column("img", col):
+                image_col = col
+                break
+        if image_col:
+            return {
+                "format": "sharegpt_with_images",
+                "needs_conversion": True,
+                "image_column": image_col,
+                "text_column": None,
+                "messages_column": chat_col,
+            }
+
     # Find image and text columns using metadata filtering
 
     # Define metadata patterns to EXCLUDE
@@ -590,7 +642,7 @@ def detect_vlm_dataset_structure(dataset):
     }
 
     # Image-related keywords
-    image_keywords = ['image', 'img', 'photo', 'picture', 'pic', 'visual', 'scan']
+    image_keywords = ['image', 'img', 'photo', 'picture', 'pic', 'visual', 'scan', 'file_name', 'filename']
 
     # Text-related keywords
     text_keywords = ['text', 'caption', 'description', 'answer', 'output', 'response', 'label']