ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: add LLM-assisted dataset detection using ephemeral GGUF helper

Browse source 
Uses Qwen2.5-3B-Instruct Q8_0 via LlamaCppBackend to complement
heuristic-based dataset detection when heuristics are uncertain.

- New llm_assist.py: VLM instruction generation, column classification,
  and user-friendly warning generation for dataset issues
- Pre-cache helper GGUF on FastAPI startup (background thread)
- Reorder training pipeline: dataset processing runs BEFORE model load
  to avoid VRAM contention (detect → dataset → model → train)
- Add pre_detect_and_load_tokenizer() for lightweight detection
- LLM warnings on VLM conversion failures (broken URLs, missing images)
- LLM column classification fallback when heuristics return unknown
- Graceful degradation: all paths unchanged when helper unavailable
This commit is contained in:
Roland Tannous 2026-03-10 09:20:45 +00:00
commit f7ca361c5c
7 changed files with 599 additions and 54 deletions
Download patch file Download diff file Expand all files Collapse all files

65
studio/backend/core/training/trainer.py
View file

@ -99,6 +99,71 @@ class UnslothTrainer:
'is_lora': True, # Default to LoRA
}
def pre_detect_and_load_tokenizer(
self,
model_name: str,
max_seq_length: int = 2048,
hf_token: Optional[str] = None,
is_dataset_image: bool = False,
is_dataset_audio: bool = False,
trust_remote_code: bool = False,
) -> None:
"""Lightweight detection and tokenizer load — no model weights, no VRAM.
Sets is_vlm, _audio_type, is_audio_vlm, model_name and loads a
lightweight tokenizer for dataset formatting. Call this before
load_and_format_dataset() when you want to process the dataset
BEFORE loading the training model (avoids VRAM contention with
the LLM-assisted detection helper).
load_model() may be called afterwards — it will re-detect and load
the full model + tokenizer, overwriting the lightweight one set here.
"""
self.model_name = model_name
self.max_seq_length = max_seq_length
self.trust_remote_code = trust_remote_code
if hf_token:
os.environ["HF_TOKEN"] = hf_token
# --- Detect audio type (reads config.json only, no VRAM) ---
self._audio_type = detect_audio_type(model_name, hf_token)
if self._audio_type == 'audio_vlm':
self.is_audio = False
self.is_audio_vlm = is_dataset_audio
self._audio_type = None
else:
self.is_audio = self._audio_type is not None
self.is_audio_vlm = False
if not self.is_audio and not self.is_audio_vlm:
self._cuda_audio_used = False
# --- Detect VLM ---
vision = is_vision_model(model_name) if not self.is_audio else False
self.is_vlm = not self.is_audio_vlm and vision and is_dataset_image
logger.info(
"pre_detect: audio_type=%s, is_audio=%s, is_audio_vlm=%s, is_vlm=%s",
self._audio_type, self.is_audio, self.is_audio_vlm, self.is_vlm,
)
# --- Load lightweight tokenizer/processor (CPU only, no VRAM) ---
# Whisper needs AutoProcessor (has feature_extractor + tokenizer).
# All others work with AutoTokenizer (CSM loads its own processor inline).
if self._audio_type == 'whisper':
from transformers import AutoProcessor
self.tokenizer = AutoProcessor.from_pretrained(
model_name, trust_remote_code=trust_remote_code, token=hf_token,
)
else:
from transformers import AutoTokenizer
self.tokenizer = AutoTokenizer.from_pretrained(
model_name, trust_remote_code=trust_remote_code, token=hf_token,
)
logger.info("Pre-loaded tokenizer for %s", model_name)
def add_progress_callback(self, callback: Callable[[TrainingProgress], None]):
"""Add callback for training progress updates"""
self.progress_callbacks.append(callback)

115
studio/backend/core/training/worker.py
View file

@ -184,68 +184,28 @@ def run_training_process(
stop_thread.start()
# ── 4. Execute the training pipeline ──
# Order: detect → dataset → model → prepare → train
# Dataset processing (including LLM-assisted detection) runs BEFORE model
# loading so both never occupy VRAM at the same time.
try:
hf_token = config.get("hf_token", "")
hf_token = hf_token if hf_token and hf_token.strip() else None
# Load model
_send_status(event_queue, "Loading model...")
success = trainer.load_model(
# ── 4a. Lightweight detection + tokenizer (no VRAM) ──
_send_status(event_queue, "Detecting model type...")
trainer.pre_detect_and_load_tokenizer(
model_name=model_name,
max_seq_length=config["max_seq_length"],
load_in_4bit=config["load_in_4bit"],
hf_token=hf_token,
is_dataset_image=config.get("is_dataset_image", False),
is_dataset_audio=config.get("is_dataset_audio", False),
trust_remote_code=config.get("trust_remote_code", False),
)
if not success or trainer.should_stop:
if trainer.should_stop:
event_queue.put({"type": "complete", "output_dir": None, "ts": time.time()})
else:
error_msg = trainer.training_progress.error or "Failed to load model"
event_queue.put({
"type": "error",
"error": error_msg,
"stack": "", "ts": time.time(),
})
if trainer.should_stop:
event_queue.put({"type": "complete", "output_dir": None, "ts": time.time()})
return
# Prepare model (LoRA or full finetuning)
training_type = config.get("training_type", "LoRA/QLoRA")
use_lora = (training_type == "LoRA/QLoRA")
if use_lora:
_send_status(event_queue, "Configuring LoRA adapters...")
success = trainer.prepare_model_for_training(
use_lora=True,
finetune_vision_layers=config.get("finetune_vision_layers", True),
finetune_language_layers=config.get("finetune_language_layers", True),
finetune_attention_modules=config.get("finetune_attention_modules", True),
finetune_mlp_modules=config.get("finetune_mlp_modules", True),
target_modules=config.get("target_modules"),
lora_r=config.get("lora_r", 16),
lora_alpha=config.get("lora_alpha", 16),
lora_dropout=config.get("lora_dropout", 0.0),
use_gradient_checkpointing=config.get("gradient_checkpointing", "unsloth"),
use_rslora=config.get("use_rslora", False),
use_loftq=config.get("use_loftq", False),
)
else:
_send_status(event_queue, "Preparing model for full finetuning...")
success = trainer.prepare_model_for_training(use_lora=False)
if not success or trainer.should_stop:
if trainer.should_stop:
event_queue.put({"type": "complete", "output_dir": None, "ts": time.time()})
else:
event_queue.put({
"type": "error",
"error": trainer.training_progress.error or "Failed to prepare model",
"stack": "", "ts": time.time(),
})
return
# Load dataset
# ── 4b. Load and format dataset (LLM helper may use VRAM briefly) ──
_send_status(event_queue, "Loading and formatting dataset...")
hf_dataset = config.get("hf_dataset", "")
dataset_result = trainer.load_and_format_dataset(
@ -291,6 +251,63 @@ def run_training_process(
})
return
# ── 4c. Load training model (uses VRAM — dataset already formatted) ──
_send_status(event_queue, "Loading model...")
success = trainer.load_model(
model_name=model_name,
max_seq_length=config["max_seq_length"],
load_in_4bit=config["load_in_4bit"],
hf_token=hf_token,
is_dataset_image=config.get("is_dataset_image", False),
is_dataset_audio=config.get("is_dataset_audio", False),
trust_remote_code=config.get("trust_remote_code", False),
)
if not success or trainer.should_stop:
if trainer.should_stop:
event_queue.put({"type": "complete", "output_dir": None, "ts": time.time()})
else:
error_msg = trainer.training_progress.error or "Failed to load model"
event_queue.put({
"type": "error",
"error": error_msg,
"stack": "", "ts": time.time(),
})
return
# ── 4d. Prepare model (LoRA or full finetuning) ──
training_type = config.get("training_type", "LoRA/QLoRA")
use_lora = (training_type == "LoRA/QLoRA")
if use_lora:
_send_status(event_queue, "Configuring LoRA adapters...")
success = trainer.prepare_model_for_training(
use_lora=True,
finetune_vision_layers=config.get("finetune_vision_layers", True),
finetune_language_layers=config.get("finetune_language_layers", True),
finetune_attention_modules=config.get("finetune_attention_modules", True),
finetune_mlp_modules=config.get("finetune_mlp_modules", True),
target_modules=config.get("target_modules"),
lora_r=config.get("lora_r", 16),
lora_alpha=config.get("lora_alpha", 16),
lora_dropout=config.get("lora_dropout", 0.0),
use_gradient_checkpointing=config.get("gradient_checkpointing", "unsloth"),
use_rslora=config.get("use_rslora", False),
use_loftq=config.get("use_loftq", False),
)
else:
_send_status(event_queue, "Preparing model for full finetuning...")
success = trainer.prepare_model_for_training(use_lora=False)
if not success or trainer.should_stop:
if trainer.should_stop:
event_queue.put({"type": "complete", "output_dir": None, "ts": time.time()})
else:
event_queue.put({
"type": "error",
"error": trainer.training_progress.error or "Failed to prepare model",
"stack": "", "ts": time.time(),
})
return
# Convert learning rate
try:
lr_value = float(config.get("learning_rate", "2e-4"))

11
studio/backend/main.py
View file

@ -60,6 +60,17 @@ async def lifespan(app: FastAPI):
f"GPU sm_{sm_version} detected — setting UNSLOTH_FLEX_ATTENTION=0"
)
# Pre-cache the helper GGUF model for LLM-assisted dataset detection.
# Runs in a background thread so it doesn't block server startup.
import threading
def _precache():
try:
from utils.datasets.llm_assist import precache_helper_gguf
precache_helper_gguf()
except Exception:
pass # non-critical
threading.Thread(target=_precache, daemon=True).start()
if not storage.is_initialized():
setup_token = secrets.token_urlsafe(32)
storage.save_setup_token(setup_token)

51
studio/backend/utils/datasets/dataset_utils.py
View file

@ -133,6 +133,41 @@ def check_dataset_format(dataset, is_vlm: bool = False) -> dict:
**audio_fields,
}
else:
# Heuristic failed — try LLM-assisted column classification
try:
from .llm_assist import llm_classify_columns
from itertools import islice
sample_rows = []
for s in islice(dataset, 5):
row = {col: str(s[col])[:200] for col in s}
sample_rows.append(row)
llm_mapping = llm_classify_columns(
column_names=columns,
samples=sample_rows,
)
if llm_mapping:
# Keep only conversation roles, not metadata
conversation_mapping = {
col: role for col, role in llm_mapping.items()
if role in ("user", "assistant", "system")
}
return {
"requires_manual_mapping": False,
"detected_format": "llm_assisted",
"columns": columns,
"suggested_mapping": conversation_mapping,
"detected_image_column": None,
"detected_text_column": None,
"is_image": False,
"multimodal_columns": None,
**audio_fields,
}
except Exception as e:
import logging
logging.getLogger(__name__).debug(f"LLM column classification skipped: {e}")
return {
"requires_manual_mapping": True,
"detected_format": "unknown",
@ -772,8 +807,22 @@ def format_and_template_dataset(
vlm_image_column = vlm_structure["image_column"]
if vlm_text_column is None or vlm_image_column is None:
columns = list(next(iter(dataset)).keys()) if dataset else []
issues = [
f"Could not auto-detect image and text columns from: {columns}",
f"VLM structure detected: {vlm_structure.get('format', 'unknown')}",
]
friendly = None
try:
from .llm_assist import llm_generate_dataset_warning
friendly = llm_generate_dataset_warning(
issues, dataset_name=dataset_name, modality="vision",
column_names=columns,
)
except Exception:
pass
errors.append(
f"Could not auto-detect image/text columns. Found: {vlm_structure}. "
friendly or f"Could not auto-detect image/text columns. Found: {vlm_structure}. "
)
return {
"dataset": dataset,

51
studio/backend/utils/datasets/format_conversion.py
View file

@ -431,7 +431,21 @@ def convert_to_vlm_format(
throughput = probe_total / probe_elapsed if probe_elapsed > 0 else 0
if fail_rate >= MAX_FAIL_RATE:
msg = (
issues = [
f"{fail_rate:.0%} of the first {PROBE_SIZE} image URLs failed to download ({probe_fail}/{probe_total})",
"Images are external URLs, not embedded in the dataset",
]
# Try LLM-friendly warning
friendly = None
try:
from .llm_assist import llm_generate_dataset_warning
friendly = llm_generate_dataset_warning(
issues, dataset_name=dataset_name, modality="vision",
column_names=[image_column, text_column],
)
except Exception:
pass
msg = friendly or (
f"⚠️ {fail_rate:.0%} of the first {PROBE_SIZE} images failed to download "
f"({probe_fail}/{probe_total}). "
"This dataset has too many broken or unreachable image URLs. "
@ -520,7 +534,20 @@ def convert_to_vlm_format(
print(f"⚠️ Skipped {failed_count}/{total} ({fail_rate:.0%}) samples with broken/unreachable images")
# For datasets that skipped the probe (small URL datasets), check fail rate now
if has_urls and fail_rate >= MAX_FAIL_RATE:
msg = (
issues = [
f"{fail_rate:.0%} of images failed to download ({failed_count}/{total})",
"Images are external URLs, not embedded in the dataset",
]
friendly = None
try:
from .llm_assist import llm_generate_dataset_warning
friendly = llm_generate_dataset_warning(
issues, dataset_name=dataset_name, modality="vision",
column_names=[image_column, text_column],
)
except Exception:
pass
msg = friendly or (
f"⚠️ {fail_rate:.0%} of images failed to download ({failed_count}/{total}). "
"This dataset has too many broken or unreachable image URLs. "
"Consider using a dataset with embedded images instead."
@ -529,9 +556,25 @@ def convert_to_vlm_format(
raise ValueError(msg)
if len(converted_list) == 0:
issues = [
f"All {total} samples failed during VLM conversion — no usable images found",
f"Image column '{image_column}' may contain URLs that are no longer accessible, "
"or local file paths that don't exist",
]
friendly = None
try:
from .llm_assist import llm_generate_dataset_warning
friendly = llm_generate_dataset_warning(
issues, dataset_name=dataset_name, modality="vision",
column_names=[image_column, text_column],
)
except Exception:
pass
raise ValueError(
f"All {total} samples failed during VLM conversion — no usable images found. "
"This dataset may contain only image URLs that are no longer accessible."
friendly or (
f"All {total} samples failed during VLM conversion — no usable images found. "
"This dataset may contain only image URLs that are no longer accessible."
)
)
print(f"✅ Converted {len(converted_list)}/{total} samples")

325
studio/backend/utils/datasets/llm_assist.py Normal file
View file

@ -0,0 +1,325 @@
# SPDX-License-Identifier: AGPL-3.0-only - See /studio/LICENSE.AGPL-3.0
# Copyright © 2025 Unsloth AI
"""
LLM-assisted dataset analysis using an ephemeral GGUF helper model.
Complements heuristic-based detection in format_detection.py and
vlm_processing.py. Only invoked when heuristics are uncertain.
Architecture:
- Instantiates LlamaCppBackend, loads model, runs completion(s), unloads.
- Not kept warm — VRAM is freed immediately after use.
- Gracefully degrades: returns None when unavailable (no binary, OOM, disabled).
"""
import json
import logging
import os
from itertools import islice
from typing import Optional
logger = logging.getLogger(__name__)
DEFAULT_HELPER_MODEL_REPO = "Qwen/Qwen2.5-3B-Instruct-GGUF"
DEFAULT_HELPER_MODEL_VARIANT = "Q8_0"
def precache_helper_gguf():
"""
Pre-download the helper GGUF to HF cache.
Called on FastAPI startup in a background thread so subsequent
``_run_with_helper()`` calls skip the download and only pay for
llama-server startup. No-op if already cached or disabled.
"""
if os.environ.get("UNSLOTH_HELPER_MODEL_DISABLE", "").strip() in ("1", "true"):
return
repo = os.environ.get("UNSLOTH_HELPER_MODEL_REPO", DEFAULT_HELPER_MODEL_REPO)
variant = os.environ.get("UNSLOTH_HELPER_MODEL_VARIANT", DEFAULT_HELPER_MODEL_VARIANT)
try:
from huggingface_hub import HfApi, hf_hub_download
# Find the GGUF file matching the variant
api = HfApi()
files = api.list_repo_files(repo, repo_type="model")
gguf_files = [f for f in files if f.endswith(".gguf")]
target = None
variant_lower = variant.lower().replace("-", "_")
for f in gguf_files:
if variant_lower in f.lower().replace("-", "_"):
target = f
break
if target:
logger.info(f"Pre-caching helper GGUF: {repo}/{target}")
hf_hub_download(repo_id=repo, filename=target)
logger.info(f"Helper GGUF cached: {target}")
else:
logger.warning(f"No GGUF matching variant '{variant}' in {repo}")
except Exception as e:
logger.warning(f"Failed to pre-cache helper GGUF: {e}")
def _run_with_helper(prompt: str, max_tokens: int = 256) -> Optional[str]:
"""
Load helper model, run one chat completion, unload.
Returns the completion text, or None on any failure.
"""
if os.environ.get("UNSLOTH_HELPER_MODEL_DISABLE", "").strip() in ("1", "true"):
return None
repo = os.environ.get("UNSLOTH_HELPER_MODEL_REPO", DEFAULT_HELPER_MODEL_REPO)
variant = os.environ.get("UNSLOTH_HELPER_MODEL_VARIANT", DEFAULT_HELPER_MODEL_VARIANT)
backend = None
try:
from core.inference.llama_cpp import LlamaCppBackend
backend = LlamaCppBackend()
logger.info(f"Loading helper model: {repo} ({variant})")
print(f"🤖 Loading helper model: {repo} ({variant})...")
ok = backend.load_model(
hf_repo=repo,
hf_variant=variant,
model_identifier=f"helper:{repo}:{variant}",
is_vision=False,
n_ctx=2048,
n_gpu_layers=-1,
)
if not ok:
logger.warning("Helper model failed to start")
return None
messages = [{"role": "user", "content": prompt}]
cumulative = ""
for text in backend.generate_chat_completion(
messages=messages,
temperature=0.1,
top_p=0.9,
top_k=20,
max_tokens=max_tokens,
repetition_penalty=1.0,
):
cumulative = text # cumulative — last value is full text
result = cumulative.strip()
logger.info(f"Helper model response ({len(result)} chars)")
return result if result else None
except Exception as e:
logger.warning(f"Helper model failed: {e}")
return None
finally:
if backend is not None:
try:
backend.unload_model()
print("🤖 Helper model unloaded")
except Exception:
pass
# ─── Public API ───────────────────────────────────────────────────────
def llm_generate_vlm_instruction(
column_names: list[str],
samples: list[dict],
dataset_name: Optional[str] = None,
) -> Optional[dict]:
"""
Ask a helper LLM to generate a task-specific VLM instruction.
Called when heuristic instruction generation returns low confidence
or falls back to generic.
Args:
column_names: Column names in the dataset.
samples: 3-5 sample rows with text values (images replaced by "<image>").
dataset_name: Optional HF dataset identifier for context.
Returns:
{"instruction": str, "confidence": 0.85} or None.
"""
# Format samples for the prompt
formatted = ""
for i, row in enumerate(samples[:5], 1):
parts = []
for col in column_names:
val = str(row.get(col, ""))[:300]
parts.append(f" {col}: {val}")
formatted += f"Sample {i}:\n" + "\n".join(parts) + "\n\n"
prompt = (
"You are a dataset analyst. Given a vision-language dataset, generate ONE "
"instruction sentence that describes what the model should do with each image.\n\n"
f"Dataset: {dataset_name or 'unknown'}\n"
f"Columns: {column_names}\n\n"
f"{formatted}"
"Write ONE instruction sentence. Examples:\n"
'- "Solve the math problem shown in the image and explain your reasoning."\n'
'- "Transcribe all text visible in this image."\n'
'- "Answer the question about this image."\n\n'
"Respond with ONLY the instruction sentence, nothing else."
)
result = _run_with_helper(prompt, max_tokens=100)
if not result:
return None
# Clean up: strip quotes, ensure it's a single sentence
instruction = result.strip().strip('"').strip("'").strip()
# Reject obviously bad outputs (too short, too long, or multi-line)
if len(instruction) < 10 or len(instruction) > 200 or "\n" in instruction:
logger.warning(f"Helper model returned unusable instruction: {instruction!r}")
return None
print(f"🤖 LLM-generated instruction: {instruction}")
return {
"instruction": instruction,
"confidence": 0.85,
}
def llm_classify_columns(
column_names: list[str],
samples: list[dict],
) -> Optional[dict[str, str]]:
"""
Ask a helper LLM to classify dataset columns into roles.
Called when heuristic column detection fails (returns None).
Args:
column_names: Column names in the dataset.
samples: 3-5 sample rows with values truncated to 200 chars.
Returns:
Dict mapping column_name → role ("user"|"assistant"|"system"|"metadata"),
or None on failure.
"""
formatted = ""
for i, row in enumerate(samples[:5], 1):
parts = []
for col in column_names:
val = str(row.get(col, ""))[:200]
parts.append(f" {col}: {val}")
formatted += f"Sample {i}:\n" + "\n".join(parts) + "\n\n"
prompt = (
"Classify each column in this dataset into one of these roles:\n"
"- user: The input/question/prompt from the human\n"
"- assistant: The expected output/answer/response from the AI\n"
"- system: Context, persona, or task description\n"
"- metadata: IDs, scores, labels, timestamps — not part of conversation\n\n"
f"Columns: {column_names}\n\n"
f"{formatted}"
"Respond with ONLY a JSON object mapping column names to roles.\n"
'Example: {"question": "user", "answer": "assistant", "id": "metadata"}'
)
result = _run_with_helper(prompt, max_tokens=200)
if not result:
return None
# Parse JSON from response (may have markdown fences)
text = result.strip()
if text.startswith("```"):
# Strip markdown code fence
lines = text.split("\n")
text = "\n".join(lines[1:-1] if lines[-1].strip() == "```" else lines[1:])
text = text.strip()
try:
mapping = json.loads(text)
except json.JSONDecodeError:
# Try to find JSON object in the response
import re
match = re.search(r"\{[^}]+\}", text)
if match:
try:
mapping = json.loads(match.group())
except json.JSONDecodeError:
logger.warning(f"Could not parse helper model JSON: {text!r}")
return None
else:
logger.warning(f"No JSON found in helper model response: {text!r}")
return None
if not isinstance(mapping, dict):
return None
# Validate: all values must be valid roles
valid_roles = {"user", "assistant", "system", "metadata"}
cleaned = {}
for col, role in mapping.items():
if col in column_names and isinstance(role, str) and role.lower() in valid_roles:
cleaned[col] = role.lower()
if not cleaned:
return None
# Must have at least user + assistant
roles_present = set(cleaned.values())
if "user" not in roles_present or "assistant" not in roles_present:
logger.warning(f"Helper model mapping missing user/assistant: {cleaned}")
return None
print(f"🤖 LLM-classified columns: {cleaned}")
return cleaned
def llm_generate_dataset_warning(
issues: list[str],
dataset_name: Optional[str] = None,
modality: str = "text",
column_names: Optional[list[str]] = None,
) -> Optional[str]:
"""
Ask the helper LLM to turn technical dataset issues into a user-friendly warning.
Works for all modalities (text, vision, audio).
Args:
issues: List of technical issue descriptions found during analysis.
dataset_name: Optional HF dataset name.
modality: "text", "vision", or "audio".
column_names: Optional list of column names for context.
Returns:
A human-friendly warning string, or None on failure.
"""
if not issues:
return None
issues_text = "\n".join(f"- {issue}" for issue in issues)
cols_text = f"\nColumns: {column_names}" if column_names else ""
prompt = (
"You are a helpful assistant. A user is trying to fine-tune a model on a dataset.\n"
"The following issues were found during dataset analysis:\n\n"
f"{issues_text}\n\n"
f"Dataset: {dataset_name or 'unknown'}\n"
f"Modality: {modality}"
f"{cols_text}\n\n"
"Write a brief, friendly explanation of what's wrong and what the user can do about it.\n"
"Keep it under 3 sentences. Be specific about the dataset."
)
result = _run_with_helper(prompt, max_tokens=200)
if not result:
return None
warning = result.strip()
# Reject obviously bad outputs
if len(warning) < 10 or len(warning) > 500:
return None
print(f"🤖 LLM-generated warning: {warning}")
return warning

37
studio/backend/utils/datasets/vlm_processing.py
View file

@ -9,6 +9,7 @@ for VLM datasets based on content analysis and heuristics.
"""
import re
from itertools import islice
def generate_smart_vlm_instruction(
@ -176,7 +177,41 @@ def generate_smart_vlm_instruction(
"confidence": 0.75,
}
# ===== LEVEL 4: Generic Fallback =====
# ===== LEVEL 4: LLM-Assisted Instruction Generation =====
try:
from .llm_assist import llm_generate_vlm_instruction
sample_rows = []
for s in islice(dataset, 5):
row = {}
for col in s:
val = s[col]
if hasattr(val, 'size') and hasattr(val, 'mode'): # PIL Image
row[col] = "<image>"
elif isinstance(val, list):
row[col] = str(val)[:300]
else:
row[col] = str(val)[:300]
sample_rows.append(row)
llm_result = llm_generate_vlm_instruction(
column_names=list(column_names),
samples=sample_rows,
dataset_name=dataset_name,
)
if llm_result and llm_result.get("instruction"):
return {
"instruction": llm_result["instruction"],
"instruction_column": None,
"instruction_type": "llm_assisted",
"uses_dynamic_instruction": False,
"confidence": llm_result.get("confidence", 0.85),
}
except Exception as e:
import logging
logging.getLogger(__name__).debug(f"LLM-assisted instruction skipped: {e}")
# ===== LEVEL 5: Generic Fallback =====
return {
"instruction": "Describe this image in detail.",
"instruction_column": None,