diff --git a/studio/backend/core/training/trainer.py b/studio/backend/core/training/trainer.py index 31e05d0bda..94f2d4d5a0 100644 --- a/studio/backend/core/training/trainer.py +++ b/studio/backend/core/training/trainer.py @@ -99,6 +99,71 @@ class UnslothTrainer: 'is_lora': True, # Default to LoRA } + def pre_detect_and_load_tokenizer( + self, + model_name: str, + max_seq_length: int = 2048, + hf_token: Optional[str] = None, + is_dataset_image: bool = False, + is_dataset_audio: bool = False, + trust_remote_code: bool = False, + ) -> None: + """Lightweight detection and tokenizer load — no model weights, no VRAM. + + Sets is_vlm, _audio_type, is_audio_vlm, model_name and loads a + lightweight tokenizer for dataset formatting. Call this before + load_and_format_dataset() when you want to process the dataset + BEFORE loading the training model (avoids VRAM contention with + the LLM-assisted detection helper). + + load_model() may be called afterwards — it will re-detect and load + the full model + tokenizer, overwriting the lightweight one set here. + """ + self.model_name = model_name + self.max_seq_length = max_seq_length + self.trust_remote_code = trust_remote_code + + if hf_token: + os.environ["HF_TOKEN"] = hf_token + + # --- Detect audio type (reads config.json only, no VRAM) --- + self._audio_type = detect_audio_type(model_name, hf_token) + if self._audio_type == 'audio_vlm': + self.is_audio = False + self.is_audio_vlm = is_dataset_audio + self._audio_type = None + else: + self.is_audio = self._audio_type is not None + self.is_audio_vlm = False + + if not self.is_audio and not self.is_audio_vlm: + self._cuda_audio_used = False + + # --- Detect VLM --- + vision = is_vision_model(model_name) if not self.is_audio else False + self.is_vlm = not self.is_audio_vlm and vision and is_dataset_image + + logger.info( + "pre_detect: audio_type=%s, is_audio=%s, is_audio_vlm=%s, is_vlm=%s", + self._audio_type, self.is_audio, self.is_audio_vlm, self.is_vlm, + ) + + # --- Load lightweight tokenizer/processor (CPU only, no VRAM) --- + # Whisper needs AutoProcessor (has feature_extractor + tokenizer). + # All others work with AutoTokenizer (CSM loads its own processor inline). + if self._audio_type == 'whisper': + from transformers import AutoProcessor + self.tokenizer = AutoProcessor.from_pretrained( + model_name, trust_remote_code=trust_remote_code, token=hf_token, + ) + else: + from transformers import AutoTokenizer + self.tokenizer = AutoTokenizer.from_pretrained( + model_name, trust_remote_code=trust_remote_code, token=hf_token, + ) + + logger.info("Pre-loaded tokenizer for %s", model_name) + def add_progress_callback(self, callback: Callable[[TrainingProgress], None]): """Add callback for training progress updates""" self.progress_callbacks.append(callback) diff --git a/studio/backend/core/training/worker.py b/studio/backend/core/training/worker.py index 7b98865eea..920ca63cfb 100644 --- a/studio/backend/core/training/worker.py +++ b/studio/backend/core/training/worker.py @@ -184,68 +184,28 @@ def run_training_process( stop_thread.start() # ── 4. Execute the training pipeline ── + # Order: detect → dataset → model → prepare → train + # Dataset processing (including LLM-assisted detection) runs BEFORE model + # loading so both never occupy VRAM at the same time. try: hf_token = config.get("hf_token", "") hf_token = hf_token if hf_token and hf_token.strip() else None - # Load model - _send_status(event_queue, "Loading model...") - success = trainer.load_model( + # ── 4a. Lightweight detection + tokenizer (no VRAM) ── + _send_status(event_queue, "Detecting model type...") + trainer.pre_detect_and_load_tokenizer( model_name=model_name, max_seq_length=config["max_seq_length"], - load_in_4bit=config["load_in_4bit"], hf_token=hf_token, is_dataset_image=config.get("is_dataset_image", False), is_dataset_audio=config.get("is_dataset_audio", False), trust_remote_code=config.get("trust_remote_code", False), ) - if not success or trainer.should_stop: - if trainer.should_stop: - event_queue.put({"type": "complete", "output_dir": None, "ts": time.time()}) - else: - error_msg = trainer.training_progress.error or "Failed to load model" - event_queue.put({ - "type": "error", - "error": error_msg, - "stack": "", "ts": time.time(), - }) + if trainer.should_stop: + event_queue.put({"type": "complete", "output_dir": None, "ts": time.time()}) return - # Prepare model (LoRA or full finetuning) - training_type = config.get("training_type", "LoRA/QLoRA") - use_lora = (training_type == "LoRA/QLoRA") - if use_lora: - _send_status(event_queue, "Configuring LoRA adapters...") - success = trainer.prepare_model_for_training( - use_lora=True, - finetune_vision_layers=config.get("finetune_vision_layers", True), - finetune_language_layers=config.get("finetune_language_layers", True), - finetune_attention_modules=config.get("finetune_attention_modules", True), - finetune_mlp_modules=config.get("finetune_mlp_modules", True), - target_modules=config.get("target_modules"), - lora_r=config.get("lora_r", 16), - lora_alpha=config.get("lora_alpha", 16), - lora_dropout=config.get("lora_dropout", 0.0), - use_gradient_checkpointing=config.get("gradient_checkpointing", "unsloth"), - use_rslora=config.get("use_rslora", False), - use_loftq=config.get("use_loftq", False), - ) - else: - _send_status(event_queue, "Preparing model for full finetuning...") - success = trainer.prepare_model_for_training(use_lora=False) - - if not success or trainer.should_stop: - if trainer.should_stop: - event_queue.put({"type": "complete", "output_dir": None, "ts": time.time()}) - else: - event_queue.put({ - "type": "error", - "error": trainer.training_progress.error or "Failed to prepare model", - "stack": "", "ts": time.time(), - }) - return - - # Load dataset + # ── 4b. Load and format dataset (LLM helper may use VRAM briefly) ── _send_status(event_queue, "Loading and formatting dataset...") hf_dataset = config.get("hf_dataset", "") dataset_result = trainer.load_and_format_dataset( @@ -291,6 +251,63 @@ def run_training_process( }) return + # ── 4c. Load training model (uses VRAM — dataset already formatted) ── + _send_status(event_queue, "Loading model...") + success = trainer.load_model( + model_name=model_name, + max_seq_length=config["max_seq_length"], + load_in_4bit=config["load_in_4bit"], + hf_token=hf_token, + is_dataset_image=config.get("is_dataset_image", False), + is_dataset_audio=config.get("is_dataset_audio", False), + trust_remote_code=config.get("trust_remote_code", False), + ) + if not success or trainer.should_stop: + if trainer.should_stop: + event_queue.put({"type": "complete", "output_dir": None, "ts": time.time()}) + else: + error_msg = trainer.training_progress.error or "Failed to load model" + event_queue.put({ + "type": "error", + "error": error_msg, + "stack": "", "ts": time.time(), + }) + return + + # ── 4d. Prepare model (LoRA or full finetuning) ── + training_type = config.get("training_type", "LoRA/QLoRA") + use_lora = (training_type == "LoRA/QLoRA") + if use_lora: + _send_status(event_queue, "Configuring LoRA adapters...") + success = trainer.prepare_model_for_training( + use_lora=True, + finetune_vision_layers=config.get("finetune_vision_layers", True), + finetune_language_layers=config.get("finetune_language_layers", True), + finetune_attention_modules=config.get("finetune_attention_modules", True), + finetune_mlp_modules=config.get("finetune_mlp_modules", True), + target_modules=config.get("target_modules"), + lora_r=config.get("lora_r", 16), + lora_alpha=config.get("lora_alpha", 16), + lora_dropout=config.get("lora_dropout", 0.0), + use_gradient_checkpointing=config.get("gradient_checkpointing", "unsloth"), + use_rslora=config.get("use_rslora", False), + use_loftq=config.get("use_loftq", False), + ) + else: + _send_status(event_queue, "Preparing model for full finetuning...") + success = trainer.prepare_model_for_training(use_lora=False) + + if not success or trainer.should_stop: + if trainer.should_stop: + event_queue.put({"type": "complete", "output_dir": None, "ts": time.time()}) + else: + event_queue.put({ + "type": "error", + "error": trainer.training_progress.error or "Failed to prepare model", + "stack": "", "ts": time.time(), + }) + return + # Convert learning rate try: lr_value = float(config.get("learning_rate", "2e-4")) diff --git a/studio/backend/main.py b/studio/backend/main.py index 3a437f08ff..44be3f8c69 100644 --- a/studio/backend/main.py +++ b/studio/backend/main.py @@ -60,6 +60,17 @@ async def lifespan(app: FastAPI): f"GPU sm_{sm_version} detected — setting UNSLOTH_FLEX_ATTENTION=0" ) + # Pre-cache the helper GGUF model for LLM-assisted dataset detection. + # Runs in a background thread so it doesn't block server startup. + import threading + def _precache(): + try: + from utils.datasets.llm_assist import precache_helper_gguf + precache_helper_gguf() + except Exception: + pass # non-critical + threading.Thread(target=_precache, daemon=True).start() + if not storage.is_initialized(): setup_token = secrets.token_urlsafe(32) storage.save_setup_token(setup_token) diff --git a/studio/backend/utils/datasets/dataset_utils.py b/studio/backend/utils/datasets/dataset_utils.py index 2b430cee14..ba2d6092f9 100644 --- a/studio/backend/utils/datasets/dataset_utils.py +++ b/studio/backend/utils/datasets/dataset_utils.py @@ -133,6 +133,41 @@ def check_dataset_format(dataset, is_vlm: bool = False) -> dict: **audio_fields, } else: + # Heuristic failed — try LLM-assisted column classification + try: + from .llm_assist import llm_classify_columns + from itertools import islice + + sample_rows = [] + for s in islice(dataset, 5): + row = {col: str(s[col])[:200] for col in s} + sample_rows.append(row) + + llm_mapping = llm_classify_columns( + column_names=columns, + samples=sample_rows, + ) + if llm_mapping: + # Keep only conversation roles, not metadata + conversation_mapping = { + col: role for col, role in llm_mapping.items() + if role in ("user", "assistant", "system") + } + return { + "requires_manual_mapping": False, + "detected_format": "llm_assisted", + "columns": columns, + "suggested_mapping": conversation_mapping, + "detected_image_column": None, + "detected_text_column": None, + "is_image": False, + "multimodal_columns": None, + **audio_fields, + } + except Exception as e: + import logging + logging.getLogger(__name__).debug(f"LLM column classification skipped: {e}") + return { "requires_manual_mapping": True, "detected_format": "unknown", @@ -772,8 +807,22 @@ def format_and_template_dataset( vlm_image_column = vlm_structure["image_column"] if vlm_text_column is None or vlm_image_column is None: + columns = list(next(iter(dataset)).keys()) if dataset else [] + issues = [ + f"Could not auto-detect image and text columns from: {columns}", + f"VLM structure detected: {vlm_structure.get('format', 'unknown')}", + ] + friendly = None + try: + from .llm_assist import llm_generate_dataset_warning + friendly = llm_generate_dataset_warning( + issues, dataset_name=dataset_name, modality="vision", + column_names=columns, + ) + except Exception: + pass errors.append( - f"Could not auto-detect image/text columns. Found: {vlm_structure}. " + friendly or f"Could not auto-detect image/text columns. Found: {vlm_structure}. " ) return { "dataset": dataset, diff --git a/studio/backend/utils/datasets/format_conversion.py b/studio/backend/utils/datasets/format_conversion.py index 7c45a03b49..0f1bb7b8fe 100644 --- a/studio/backend/utils/datasets/format_conversion.py +++ b/studio/backend/utils/datasets/format_conversion.py @@ -431,7 +431,21 @@ def convert_to_vlm_format( throughput = probe_total / probe_elapsed if probe_elapsed > 0 else 0 if fail_rate >= MAX_FAIL_RATE: - msg = ( + issues = [ + f"{fail_rate:.0%} of the first {PROBE_SIZE} image URLs failed to download ({probe_fail}/{probe_total})", + "Images are external URLs, not embedded in the dataset", + ] + # Try LLM-friendly warning + friendly = None + try: + from .llm_assist import llm_generate_dataset_warning + friendly = llm_generate_dataset_warning( + issues, dataset_name=dataset_name, modality="vision", + column_names=[image_column, text_column], + ) + except Exception: + pass + msg = friendly or ( f"⚠️ {fail_rate:.0%} of the first {PROBE_SIZE} images failed to download " f"({probe_fail}/{probe_total}). " "This dataset has too many broken or unreachable image URLs. " @@ -520,7 +534,20 @@ def convert_to_vlm_format( print(f"⚠️ Skipped {failed_count}/{total} ({fail_rate:.0%}) samples with broken/unreachable images") # For datasets that skipped the probe (small URL datasets), check fail rate now if has_urls and fail_rate >= MAX_FAIL_RATE: - msg = ( + issues = [ + f"{fail_rate:.0%} of images failed to download ({failed_count}/{total})", + "Images are external URLs, not embedded in the dataset", + ] + friendly = None + try: + from .llm_assist import llm_generate_dataset_warning + friendly = llm_generate_dataset_warning( + issues, dataset_name=dataset_name, modality="vision", + column_names=[image_column, text_column], + ) + except Exception: + pass + msg = friendly or ( f"⚠️ {fail_rate:.0%} of images failed to download ({failed_count}/{total}). " "This dataset has too many broken or unreachable image URLs. " "Consider using a dataset with embedded images instead." @@ -529,9 +556,25 @@ def convert_to_vlm_format( raise ValueError(msg) if len(converted_list) == 0: + issues = [ + f"All {total} samples failed during VLM conversion — no usable images found", + f"Image column '{image_column}' may contain URLs that are no longer accessible, " + "or local file paths that don't exist", + ] + friendly = None + try: + from .llm_assist import llm_generate_dataset_warning + friendly = llm_generate_dataset_warning( + issues, dataset_name=dataset_name, modality="vision", + column_names=[image_column, text_column], + ) + except Exception: + pass raise ValueError( - f"All {total} samples failed during VLM conversion — no usable images found. " - "This dataset may contain only image URLs that are no longer accessible." + friendly or ( + f"All {total} samples failed during VLM conversion — no usable images found. " + "This dataset may contain only image URLs that are no longer accessible." + ) ) print(f"✅ Converted {len(converted_list)}/{total} samples") diff --git a/studio/backend/utils/datasets/llm_assist.py b/studio/backend/utils/datasets/llm_assist.py new file mode 100644 index 0000000000..7ac25272fc --- /dev/null +++ b/studio/backend/utils/datasets/llm_assist.py @@ -0,0 +1,325 @@ +# SPDX-License-Identifier: AGPL-3.0-only - See /studio/LICENSE.AGPL-3.0 +# Copyright © 2025 Unsloth AI + +""" +LLM-assisted dataset analysis using an ephemeral GGUF helper model. + +Complements heuristic-based detection in format_detection.py and +vlm_processing.py. Only invoked when heuristics are uncertain. + +Architecture: + - Instantiates LlamaCppBackend, loads model, runs completion(s), unloads. + - Not kept warm — VRAM is freed immediately after use. + - Gracefully degrades: returns None when unavailable (no binary, OOM, disabled). +""" + +import json +import logging +import os +from itertools import islice +from typing import Optional + +logger = logging.getLogger(__name__) + +DEFAULT_HELPER_MODEL_REPO = "Qwen/Qwen2.5-3B-Instruct-GGUF" +DEFAULT_HELPER_MODEL_VARIANT = "Q8_0" + + +def precache_helper_gguf(): + """ + Pre-download the helper GGUF to HF cache. + + Called on FastAPI startup in a background thread so subsequent + ``_run_with_helper()`` calls skip the download and only pay for + llama-server startup. No-op if already cached or disabled. + """ + if os.environ.get("UNSLOTH_HELPER_MODEL_DISABLE", "").strip() in ("1", "true"): + return + + repo = os.environ.get("UNSLOTH_HELPER_MODEL_REPO", DEFAULT_HELPER_MODEL_REPO) + variant = os.environ.get("UNSLOTH_HELPER_MODEL_VARIANT", DEFAULT_HELPER_MODEL_VARIANT) + + try: + from huggingface_hub import HfApi, hf_hub_download + + # Find the GGUF file matching the variant + api = HfApi() + files = api.list_repo_files(repo, repo_type="model") + gguf_files = [f for f in files if f.endswith(".gguf")] + + target = None + variant_lower = variant.lower().replace("-", "_") + for f in gguf_files: + if variant_lower in f.lower().replace("-", "_"): + target = f + break + + if target: + logger.info(f"Pre-caching helper GGUF: {repo}/{target}") + hf_hub_download(repo_id=repo, filename=target) + logger.info(f"Helper GGUF cached: {target}") + else: + logger.warning(f"No GGUF matching variant '{variant}' in {repo}") + except Exception as e: + logger.warning(f"Failed to pre-cache helper GGUF: {e}") + + +def _run_with_helper(prompt: str, max_tokens: int = 256) -> Optional[str]: + """ + Load helper model, run one chat completion, unload. + + Returns the completion text, or None on any failure. + """ + if os.environ.get("UNSLOTH_HELPER_MODEL_DISABLE", "").strip() in ("1", "true"): + return None + + repo = os.environ.get("UNSLOTH_HELPER_MODEL_REPO", DEFAULT_HELPER_MODEL_REPO) + variant = os.environ.get("UNSLOTH_HELPER_MODEL_VARIANT", DEFAULT_HELPER_MODEL_VARIANT) + + backend = None + try: + from core.inference.llama_cpp import LlamaCppBackend + + backend = LlamaCppBackend() + logger.info(f"Loading helper model: {repo} ({variant})") + print(f"🤖 Loading helper model: {repo} ({variant})...") + + ok = backend.load_model( + hf_repo=repo, + hf_variant=variant, + model_identifier=f"helper:{repo}:{variant}", + is_vision=False, + n_ctx=2048, + n_gpu_layers=-1, + ) + if not ok: + logger.warning("Helper model failed to start") + return None + + messages = [{"role": "user", "content": prompt}] + cumulative = "" + for text in backend.generate_chat_completion( + messages=messages, + temperature=0.1, + top_p=0.9, + top_k=20, + max_tokens=max_tokens, + repetition_penalty=1.0, + ): + cumulative = text # cumulative — last value is full text + + result = cumulative.strip() + logger.info(f"Helper model response ({len(result)} chars)") + return result if result else None + + except Exception as e: + logger.warning(f"Helper model failed: {e}") + return None + + finally: + if backend is not None: + try: + backend.unload_model() + print("🤖 Helper model unloaded") + except Exception: + pass + + +# ─── Public API ─────────────────────────────────────────────────────── + + +def llm_generate_vlm_instruction( + column_names: list[str], + samples: list[dict], + dataset_name: Optional[str] = None, +) -> Optional[dict]: + """ + Ask a helper LLM to generate a task-specific VLM instruction. + + Called when heuristic instruction generation returns low confidence + or falls back to generic. + + Args: + column_names: Column names in the dataset. + samples: 3-5 sample rows with text values (images replaced by ""). + dataset_name: Optional HF dataset identifier for context. + + Returns: + {"instruction": str, "confidence": 0.85} or None. + """ + # Format samples for the prompt + formatted = "" + for i, row in enumerate(samples[:5], 1): + parts = [] + for col in column_names: + val = str(row.get(col, ""))[:300] + parts.append(f" {col}: {val}") + formatted += f"Sample {i}:\n" + "\n".join(parts) + "\n\n" + + prompt = ( + "You are a dataset analyst. Given a vision-language dataset, generate ONE " + "instruction sentence that describes what the model should do with each image.\n\n" + f"Dataset: {dataset_name or 'unknown'}\n" + f"Columns: {column_names}\n\n" + f"{formatted}" + "Write ONE instruction sentence. Examples:\n" + '- "Solve the math problem shown in the image and explain your reasoning."\n' + '- "Transcribe all text visible in this image."\n' + '- "Answer the question about this image."\n\n' + "Respond with ONLY the instruction sentence, nothing else." + ) + + result = _run_with_helper(prompt, max_tokens=100) + if not result: + return None + + # Clean up: strip quotes, ensure it's a single sentence + instruction = result.strip().strip('"').strip("'").strip() + # Reject obviously bad outputs (too short, too long, or multi-line) + if len(instruction) < 10 or len(instruction) > 200 or "\n" in instruction: + logger.warning(f"Helper model returned unusable instruction: {instruction!r}") + return None + + print(f"🤖 LLM-generated instruction: {instruction}") + return { + "instruction": instruction, + "confidence": 0.85, + } + + +def llm_classify_columns( + column_names: list[str], + samples: list[dict], +) -> Optional[dict[str, str]]: + """ + Ask a helper LLM to classify dataset columns into roles. + + Called when heuristic column detection fails (returns None). + + Args: + column_names: Column names in the dataset. + samples: 3-5 sample rows with values truncated to 200 chars. + + Returns: + Dict mapping column_name → role ("user"|"assistant"|"system"|"metadata"), + or None on failure. + """ + formatted = "" + for i, row in enumerate(samples[:5], 1): + parts = [] + for col in column_names: + val = str(row.get(col, ""))[:200] + parts.append(f" {col}: {val}") + formatted += f"Sample {i}:\n" + "\n".join(parts) + "\n\n" + + prompt = ( + "Classify each column in this dataset into one of these roles:\n" + "- user: The input/question/prompt from the human\n" + "- assistant: The expected output/answer/response from the AI\n" + "- system: Context, persona, or task description\n" + "- metadata: IDs, scores, labels, timestamps — not part of conversation\n\n" + f"Columns: {column_names}\n\n" + f"{formatted}" + "Respond with ONLY a JSON object mapping column names to roles.\n" + 'Example: {"question": "user", "answer": "assistant", "id": "metadata"}' + ) + + result = _run_with_helper(prompt, max_tokens=200) + if not result: + return None + + # Parse JSON from response (may have markdown fences) + text = result.strip() + if text.startswith("```"): + # Strip markdown code fence + lines = text.split("\n") + text = "\n".join(lines[1:-1] if lines[-1].strip() == "```" else lines[1:]) + text = text.strip() + + try: + mapping = json.loads(text) + except json.JSONDecodeError: + # Try to find JSON object in the response + import re + match = re.search(r"\{[^}]+\}", text) + if match: + try: + mapping = json.loads(match.group()) + except json.JSONDecodeError: + logger.warning(f"Could not parse helper model JSON: {text!r}") + return None + else: + logger.warning(f"No JSON found in helper model response: {text!r}") + return None + + if not isinstance(mapping, dict): + return None + + # Validate: all values must be valid roles + valid_roles = {"user", "assistant", "system", "metadata"} + cleaned = {} + for col, role in mapping.items(): + if col in column_names and isinstance(role, str) and role.lower() in valid_roles: + cleaned[col] = role.lower() + + if not cleaned: + return None + + # Must have at least user + assistant + roles_present = set(cleaned.values()) + if "user" not in roles_present or "assistant" not in roles_present: + logger.warning(f"Helper model mapping missing user/assistant: {cleaned}") + return None + + print(f"🤖 LLM-classified columns: {cleaned}") + return cleaned + + +def llm_generate_dataset_warning( + issues: list[str], + dataset_name: Optional[str] = None, + modality: str = "text", + column_names: Optional[list[str]] = None, +) -> Optional[str]: + """ + Ask the helper LLM to turn technical dataset issues into a user-friendly warning. + + Works for all modalities (text, vision, audio). + + Args: + issues: List of technical issue descriptions found during analysis. + dataset_name: Optional HF dataset name. + modality: "text", "vision", or "audio". + column_names: Optional list of column names for context. + + Returns: + A human-friendly warning string, or None on failure. + """ + if not issues: + return None + + issues_text = "\n".join(f"- {issue}" for issue in issues) + cols_text = f"\nColumns: {column_names}" if column_names else "" + + prompt = ( + "You are a helpful assistant. A user is trying to fine-tune a model on a dataset.\n" + "The following issues were found during dataset analysis:\n\n" + f"{issues_text}\n\n" + f"Dataset: {dataset_name or 'unknown'}\n" + f"Modality: {modality}" + f"{cols_text}\n\n" + "Write a brief, friendly explanation of what's wrong and what the user can do about it.\n" + "Keep it under 3 sentences. Be specific about the dataset." + ) + + result = _run_with_helper(prompt, max_tokens=200) + if not result: + return None + + warning = result.strip() + # Reject obviously bad outputs + if len(warning) < 10 or len(warning) > 500: + return None + + print(f"🤖 LLM-generated warning: {warning}") + return warning diff --git a/studio/backend/utils/datasets/vlm_processing.py b/studio/backend/utils/datasets/vlm_processing.py index b17c57ef35..0d29a45400 100644 --- a/studio/backend/utils/datasets/vlm_processing.py +++ b/studio/backend/utils/datasets/vlm_processing.py @@ -9,6 +9,7 @@ for VLM datasets based on content analysis and heuristics. """ import re +from itertools import islice def generate_smart_vlm_instruction( @@ -176,7 +177,41 @@ def generate_smart_vlm_instruction( "confidence": 0.75, } - # ===== LEVEL 4: Generic Fallback ===== + # ===== LEVEL 4: LLM-Assisted Instruction Generation ===== + try: + from .llm_assist import llm_generate_vlm_instruction + + sample_rows = [] + for s in islice(dataset, 5): + row = {} + for col in s: + val = s[col] + if hasattr(val, 'size') and hasattr(val, 'mode'): # PIL Image + row[col] = "" + elif isinstance(val, list): + row[col] = str(val)[:300] + else: + row[col] = str(val)[:300] + sample_rows.append(row) + + llm_result = llm_generate_vlm_instruction( + column_names=list(column_names), + samples=sample_rows, + dataset_name=dataset_name, + ) + if llm_result and llm_result.get("instruction"): + return { + "instruction": llm_result["instruction"], + "instruction_column": None, + "instruction_type": "llm_assisted", + "uses_dynamic_instruction": False, + "confidence": llm_result.get("confidence", 0.85), + } + except Exception as e: + import logging + logging.getLogger(__name__).debug(f"LLM-assisted instruction skipped: {e}") + + # ===== LEVEL 5: Generic Fallback ===== return { "instruction": "Describe this image in detail.", "instruction_column": None,