From c3c65cded804f461e39dc79cd94222aa13c33f01 Mon Sep 17 00:00:00 2001 From: Shine1i Date: Sun, 1 Mar 2026 10:57:51 +0100 Subject: [PATCH] feat(recipe-studio): add image preview support for dataset and LLM configurations p1 --- .../backend/core/data_recipe/jobs/manager.py | 6 +- .../backend/core/data_recipe/jobs/worker.py | 4 +- studio/backend/core/data_recipe/jsonable.py | 40 ++++++ studio/backend/routes/data_recipe/seed.py | 27 ++-- .../executions/execution-data-tab.tsx | 3 + .../components/executions/executions-view.tsx | 24 ++++ .../recipe-studio/dialogs/llm/general-tab.tsx | 100 ++++++++++++++ .../dialogs/seed/seed-dialog.tsx | 44 ++++-- .../src/features/recipe-studio/types/index.ts | 9 ++ .../recipe-studio/utils/config-factories.ts | 6 + .../recipe-studio/utils/image-preview.ts | 126 ++++++++++++++++++ .../utils/import/parsers/llm-parser.ts | 22 +++ .../utils/payload/build-payload.ts | 25 ++++ .../utils/payload/builders-llm.ts | 24 ++++ .../recipe-studio/utils/validation.ts | 5 + 15 files changed, 432 insertions(+), 33 deletions(-) create mode 100644 studio/frontend/src/features/recipe-studio/utils/image-preview.ts diff --git a/studio/backend/core/data_recipe/jobs/manager.py b/studio/backend/core/data_recipe/jobs/manager.py index 2a8b1fe44d..de367dcce6 100644 --- a/studio/backend/core/data_recipe/jobs/manager.py +++ b/studio/backend/core/data_recipe/jobs/manager.py @@ -13,7 +13,7 @@ from typing import Any import multiprocessing as mp -from ..jsonable import to_jsonable +from ..jsonable import to_preview_jsonable from .constants import ( EVENT_JOB_CANCELLING, EVENT_JOB_CANCELLED, @@ -299,7 +299,7 @@ class JobManager: dataframe = dataframe.drop(columns=[helper_col]) rows = dataframe.to_dict(orient="records") - return {"dataset": to_jsonable(rows), "total": total} + return {"dataset": to_preview_jsonable(rows), "total": total} @staticmethod def _load_dataset_page_with_data_designer( @@ -313,7 +313,7 @@ class JobManager: dataframe = read_parquet_dataset(parquet_dir) total = int(len(dataframe.index)) rows = dataframe.iloc[offset:offset + limit].to_dict(orient="records") - return {"dataset": to_jsonable(rows), "total": total} + return {"dataset": to_preview_jsonable(rows), "total": total} def subscribe(self, job_id: str, *, after_seq: int | None = None) -> Subscription | None: """SSE subscribe: get replay buffer + live events stream.""" diff --git a/studio/backend/core/data_recipe/jobs/worker.py b/studio/backend/core/data_recipe/jobs/worker.py index 8c0996b140..d8f744868c 100644 --- a/studio/backend/core/data_recipe/jobs/worker.py +++ b/studio/backend/core/data_recipe/jobs/worker.py @@ -7,7 +7,7 @@ import traceback from pathlib import Path from typing import Any -from ..jsonable import to_jsonable +from ..jsonable import to_jsonable, to_preview_jsonable from .constants import EVENT_JOB_COMPLETED, EVENT_JOB_ERROR, EVENT_JOB_STARTED from ..service import build_config_builder, create_data_designer @@ -84,7 +84,7 @@ def run_job_process( dataset = ( [] if results.dataset is None - else to_jsonable(results.dataset.to_dict(orient="records")) + else to_preview_jsonable(results.dataset.to_dict(orient="records")) ) processor_artifacts = ( None diff --git a/studio/backend/core/data_recipe/jsonable.py b/studio/backend/core/data_recipe/jsonable.py index aa6e1d6b2e..a6828b2274 100644 --- a/studio/backend/core/data_recipe/jsonable.py +++ b/studio/backend/core/data_recipe/jsonable.py @@ -1,5 +1,7 @@ from __future__ import annotations +import base64 +import io from typing import Any @@ -28,3 +30,41 @@ def to_jsonable(value: Any) -> Any: return value return value + + +def _to_preview_image_payload(value: Any) -> dict[str, Any] | None: + try: + from PIL.Image import Image as PILImage # type: ignore + except ImportError: # pragma: no cover + return None + + if not isinstance(value, PILImage): + return None + + buffer = io.BytesIO() + value.convert("RGB").save(buffer, format="JPEG", quality=85) + return { + "type": "image", + "mime": "image/jpeg", + "width": value.width, + "height": value.height, + "data": base64.b64encode(buffer.getvalue()).decode("ascii"), + } + + +def to_preview_jsonable(value: Any) -> Any: + """Convert values into JSON-safe preview values, including PIL images.""" + image_payload = _to_preview_image_payload(value) + if image_payload is not None: + return image_payload + + converted = to_jsonable(value) + if converted is None or isinstance(converted, (str, int, float, bool)): + return converted + if isinstance(converted, dict): + return {str(k): to_preview_jsonable(v) for k, v in converted.items()} + if isinstance(converted, (list, tuple, set)): + return [to_preview_jsonable(v) for v in converted] + if isinstance(converted, (bytes, bytearray)): + return base64.b64encode(bytes(converted)).decode("ascii") + return str(converted) diff --git a/studio/backend/routes/data_recipe/seed.py b/studio/backend/routes/data_recipe/seed.py index eb02ab2bbf..1b8df3bc9b 100644 --- a/studio/backend/routes/data_recipe/seed.py +++ b/studio/backend/routes/data_recipe/seed.py @@ -10,6 +10,7 @@ from typing import Any from uuid import uuid4 from fastapi import APIRouter, HTTPException +from core.data_recipe.jsonable import to_preview_jsonable from models.data_recipe import ( SeedInspectRequest, @@ -26,13 +27,7 @@ SEED_UPLOAD_DIR = Path.home() / ".cache" / "unsloth" / "data-recipe" / "seed-upl def _serialize_preview_value(value: Any) -> Any: - if value is None or isinstance(value, (str, int, float, bool)): - return value - if isinstance(value, dict): - return {str(key): _serialize_preview_value(item) for key, item in value.items()} - if isinstance(value, (list, tuple)): - return [_serialize_preview_value(item) for item in value] - return str(value) + return to_preview_jsonable(value) def _serialize_preview_rows(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: @@ -63,10 +58,10 @@ def _list_hf_data_files(*, dataset_name: str, token: str | None) -> list[str]: return [] -def _select_best_file(data_files: list[str]) -> str | None: +def _select_best_file(data_files: list[str], split: str = DEFAULT_SPLIT) -> str | None: if not data_files: return None - split_lower = DEFAULT_SPLIT + split_lower = split.lower() def score(path: str) -> tuple[int, int]: name = path.lower() @@ -85,8 +80,8 @@ def _select_best_file(data_files: list[str]) -> str | None: return sorted(data_files, key=score)[0] -def _resolve_seed_hf_path(dataset_name: str, data_files: list[str]) -> str | None: - selected = _select_best_file(data_files) +def _resolve_seed_hf_path(dataset_name: str, data_files: list[str], split: str = DEFAULT_SPLIT) -> str | None: + selected = _select_best_file(data_files, split) if not selected: return None @@ -196,7 +191,7 @@ def inspect_seed_dataset(payload: SeedInspectRequest) -> SeedInspectResponse: except ImportError as exc: raise HTTPException(status_code=500, detail=f"seed inspect dependencies unavailable: {exc}") from exc - split = DEFAULT_SPLIT + split = _normalize_optional_text(payload.split) or DEFAULT_SPLIT subset = _normalize_optional_text(payload.subset) token = _normalize_optional_text(payload.hf_token) preview_size = int(payload.preview_size) @@ -204,12 +199,12 @@ def inspect_seed_dataset(payload: SeedInspectRequest) -> SeedInspectResponse: preview_rows: list[dict[str, Any]] = [] data_files = _list_hf_data_files(dataset_name=dataset_name, token=token) - selected_file = _select_best_file(data_files) + selected_file = _select_best_file(data_files, split) if selected_file: try: single_file_kwargs = _build_stream_load_kwargs( dataset_name=dataset_name, - split=DEFAULT_SPLIT, + split=split, subset=subset, token=token, data_file=selected_file, @@ -246,7 +241,7 @@ def inspect_seed_dataset(payload: SeedInspectRequest) -> SeedInspectResponse: if not data_files: resolved_path = f"datasets/{dataset_name}/**/*.parquet" else: - resolved_path = _resolve_seed_hf_path(dataset_name, data_files) + resolved_path = _resolve_seed_hf_path(dataset_name, data_files, split) if not resolved_path: raise HTTPException(status_code=422, detail="unable to resolve seed dataset path") @@ -255,7 +250,7 @@ def inspect_seed_dataset(payload: SeedInspectRequest) -> SeedInspectResponse: resolved_path=resolved_path, columns=columns, preview_rows=preview_rows, - split=None, + split=split, subset=subset, ) diff --git a/studio/frontend/src/features/recipe-studio/components/executions/execution-data-tab.tsx b/studio/frontend/src/features/recipe-studio/components/executions/execution-data-tab.tsx index 56ca1d5057..e68a047c85 100644 --- a/studio/frontend/src/features/recipe-studio/components/executions/execution-data-tab.tsx +++ b/studio/frontend/src/features/recipe-studio/components/executions/execution-data-tab.tsx @@ -10,6 +10,7 @@ import { DropdownMenuTrigger, } from "@/components/ui/dropdown-menu"; import { cn } from "@/lib/utils"; +import { resolveImagePreview } from "../../utils/image-preview"; import { isExecutionInProgress } from "../../executions/execution-helpers"; import type { RecipeExecutionRecord } from "../../execution-types"; import { formatCellValue, isExpandableCellValue } from "./executions-view-helpers"; @@ -130,6 +131,7 @@ export function ExecutionDataTab({ data={datasetRowsForTable} getRowClassName={(row, _rowIndex, rowId) => { const canExpand = visibleDatasetColumnNames.some((columnName) => + !resolveImagePreview(row[columnName]) && isExpandableCellValue(formatCellValue(row[columnName])), ); if (!canExpand) { @@ -142,6 +144,7 @@ export function ExecutionDataTab({ }} onRowClick={(row, _rowIndex, rowId) => { const canExpand = visibleDatasetColumnNames.some((columnName) => + !resolveImagePreview(row[columnName]) && isExpandableCellValue(formatCellValue(row[columnName])), ); if (!canExpand || !selectedExecutionIdSafe) { diff --git a/studio/frontend/src/features/recipe-studio/components/executions/executions-view.tsx b/studio/frontend/src/features/recipe-studio/components/executions/executions-view.tsx index 4deeb92982..6ebfa6af5b 100644 --- a/studio/frontend/src/features/recipe-studio/components/executions/executions-view.tsx +++ b/studio/frontend/src/features/recipe-studio/components/executions/executions-view.tsx @@ -10,6 +10,7 @@ import { Button } from "@/components/ui/button"; import { Progress } from "@/components/ui/progress"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { cn } from "@/lib/utils"; +import { resolveImagePreview } from "../../utils/image-preview"; import type { RecipeExecutionRecord, } from "../../execution-types"; @@ -119,9 +120,32 @@ export function ExecutionsView({ header: name, cell: ({ getValue, row }) => { const rawValue = getValue(); + const imagePreview = resolveImagePreview(rawValue); + if (imagePreview?.kind === "ready") { + return ( +
+ {`${name} +
+ ); + } + if (imagePreview?.kind === "too_large") { + return ( +
+

+ Image too large to preview +

+
+ ); + } const value = formatCellValue(rawValue); const rowExpanded = Boolean(expandedDatasetRows[row.id]); const rowHasExpandableCell = visibleDatasetColumnNames.some((columnName) => + !resolveImagePreview(row.original[columnName]) && isExpandableCellValue(formatCellValue(row.original[columnName])), ); const showTruncated = rowHasExpandableCell && !rowExpanded; diff --git a/studio/frontend/src/features/recipe-studio/dialogs/llm/general-tab.tsx b/studio/frontend/src/features/recipe-studio/dialogs/llm/general-tab.tsx index 1fbf267d47..547e8745ce 100644 --- a/studio/frontend/src/features/recipe-studio/dialogs/llm/general-tab.tsx +++ b/studio/frontend/src/features/recipe-studio/dialogs/llm/general-tab.tsx @@ -6,6 +6,7 @@ import { ComboboxItem, ComboboxList, } from "@/components/ui/combobox"; +import { Switch } from "@/components/ui/switch"; import { Select, SelectContent, @@ -16,6 +17,7 @@ import { import { Textarea } from "@/components/ui/textarea"; import { type ReactElement, type RefObject, useMemo } from "react"; import { useRecipeStudioStore } from "../../stores/recipe-studio"; +import { isLikelyImageValue } from "../../utils/image-preview"; import type { LlmConfig } from "../../types"; import { findInvalidJinjaReferences } from "../../utils/refs"; import { getAvailableVariables } from "../../utils/variables"; @@ -85,6 +87,39 @@ export function LlmGeneralTab({ .slice(0, 3) .map((ref) => `{{ ${ref} }}`) .join(", "); + const seedConfig = useMemo( + () => Object.values(configs).find((item) => item.kind === "seed"), + [configs], + ); + const seedColumns = seedConfig?.seed_columns ?? []; + const seedPreviewRows = seedConfig?.seed_preview_rows ?? []; + const imageColumnOptions = useMemo(() => { + if (seedColumns.length === 0) { + return []; + } + const detected = seedColumns.filter((columnName) => { + const lower = columnName.toLowerCase(); + if ( + lower.includes("image") || + lower.includes("img") || + lower.includes("photo") || + lower.includes("picture") || + lower.includes("base64") || + lower.includes("url") + ) { + return true; + } + return seedPreviewRows.some((row) => isLikelyImageValue(row[columnName])); + }); + return detected.length > 0 ? detected : seedColumns; + }, [seedColumns, seedPreviewRows]); + const imageContext = config.image_context ?? { + enabled: false, + // biome-ignore lint/style/useNamingConvention: api schema + column_name: "", + }; + const imageContextToggleId = `${config.id}-image-context-enabled`; + const imageContextColumnId = `${config.id}-image-context-column`; return (
@@ -185,6 +220,71 @@ export function LlmGeneralTab({

)}
+
+
+
+ + {imageColumnOptions.length > 0 && ( +

+ Suggested image columns: {imageColumnOptions.join(", ")} +

+ )} +
+ { + onUpdate({ + image_context: { + ...imageContext, + enabled: checked, + // biome-ignore lint/style/useNamingConvention: api schema + column_name: + checked && !imageContext.column_name + ? (imageColumnOptions[0] ?? "") + : imageContext.column_name, + }, + }); + }} + /> +
+ {imageContext.enabled && ( +
+ + +
+ )} +
{config.llm_type === "structured" && (
new Set(selectedSeedDropColumns), [selectedSeedDropColumns], ); + const rowHasExpandableText = useCallback( + (row: Record): boolean => + previewColumns.some((columnName) => { + if (resolveImagePreview(row[columnName])) { + return false; + } + return isExpandablePreviewValue(stringifyCell(row[columnName])); + }), + [previewColumns], + ); return ( @@ -778,15 +790,11 @@ export function SeedDialog({ config, onUpdate, open }: SeedDialogProps): ReactEl - isExpandablePreviewValue(stringifyCell(row[col])), - ) && "cursor-pointer hover:bg-primary/[0.06]", + rowHasExpandableText(row) && "cursor-pointer hover:bg-primary/[0.06]", expandedPreviewRows[rowIdx] && "bg-primary/[0.05]", )} onClick={() => { - const canExpand = previewColumns.some((col) => - isExpandablePreviewValue(stringifyCell(row[col])), - ); + const canExpand = rowHasExpandableText(row); if (!canExpand) { return; } @@ -802,10 +810,22 @@ export function SeedDialog({ config, onUpdate, open }: SeedDialogProps): ReactEl className="max-w-[260px] whitespace-pre-wrap break-words text-xs" > {(() => { + const imagePreview = resolveImagePreview(row[col]); + if (imagePreview?.kind === "ready") { + return ( + {`${col} + ); + } + if (imagePreview?.kind === "too_large") { + return "Image too large to preview"; + } const value = stringifyCell(row[col]); - const rowHasExpandableCell = previewColumns.some((columnName) => - isExpandablePreviewValue(stringifyCell(row[columnName])), - ); + const rowHasExpandableCell = rowHasExpandableText(row); const rowExpanded = Boolean(expandedPreviewRows[rowIdx]); return rowHasExpandableCell && !rowExpanded ? truncatePreviewValue(value) diff --git a/studio/frontend/src/features/recipe-studio/types/index.ts b/studio/frontend/src/features/recipe-studio/types/index.ts index 7d36de28cd..d12002edcc 100644 --- a/studio/frontend/src/features/recipe-studio/types/index.ts +++ b/studio/frontend/src/features/recipe-studio/types/index.ts @@ -152,6 +152,12 @@ export type LlmToolConfig = { timeout_sec?: string; }; +export type LlmImageContextConfig = { + enabled: boolean; + // biome-ignore lint/style/useNamingConvention: api schema + column_name: string; +}; + export type LlmConfig = { id: string; kind: "llm"; @@ -175,6 +181,9 @@ export type LlmConfig = { // biome-ignore lint/style/useNamingConvention: ui schema mcp_providers?: LlmMcpProviderConfig[]; scores?: Score[]; + // ui-only, serialized into multi_modal_context for DataDesigner + // biome-ignore lint/style/useNamingConvention: ui schema + image_context?: LlmImageContextConfig; }; export type ModelProviderConfig = { diff --git a/studio/frontend/src/features/recipe-studio/utils/config-factories.ts b/studio/frontend/src/features/recipe-studio/utils/config-factories.ts index f92589b213..9b711c91f5 100644 --- a/studio/frontend/src/features/recipe-studio/utils/config-factories.ts +++ b/studio/frontend/src/features/recipe-studio/utils/config-factories.ts @@ -204,6 +204,12 @@ export function makeLlmConfig( tool_configs: [], // biome-ignore lint/style/useNamingConvention: ui schema mcp_providers: [], + // biome-ignore lint/style/useNamingConvention: ui schema + image_context: { + enabled: false, + // biome-ignore lint/style/useNamingConvention: api schema + column_name: "", + }, scores: llmType === "judge" ? [ diff --git a/studio/frontend/src/features/recipe-studio/utils/image-preview.ts b/studio/frontend/src/features/recipe-studio/utils/image-preview.ts new file mode 100644 index 0000000000..61099b8f4a --- /dev/null +++ b/studio/frontend/src/features/recipe-studio/utils/image-preview.ts @@ -0,0 +1,126 @@ +export const MAX_IMAGE_PREVIEW_BYTES = 200 * 1024; + +type PreviewImagePayload = { + type?: unknown; + mime?: unknown; + data?: unknown; +}; + +export type ImagePreviewResult = + | { kind: "ready"; src: string } + | { kind: "too_large"; estimatedBytes: number }; + +function normalizeBase64(value: string): string { + return value.replace(/\s+/g, ""); +} + +function estimateBase64Bytes(base64: string): number { + const normalized = normalizeBase64(base64); + const padding = normalized.endsWith("==") + ? 2 + : normalized.endsWith("=") + ? 1 + : 0; + return Math.max(0, Math.floor((normalized.length * 3) / 4) - padding); +} + +function inferMimeFromBase64(base64: string): string | null { + const normalized = normalizeBase64(base64); + if (normalized.startsWith("iVBORw0KGgo")) { + return "image/png"; + } + if (normalized.startsWith("/9j/")) { + return "image/jpeg"; + } + if (normalized.startsWith("R0lGOD")) { + return "image/gif"; + } + if (normalized.startsWith("UklGR")) { + return "image/webp"; + } + return null; +} + +function isLikelyRawBase64Image(value: string): boolean { + const normalized = normalizeBase64(value); + if (normalized.length < 64) { + return false; + } + if (!/^[A-Za-z0-9+/=]+$/.test(normalized)) { + return false; + } + return inferMimeFromBase64(normalized) !== null; +} + +function toDataUrlFromBase64(base64: string, mime: string): string { + return `data:${mime};base64,${normalizeBase64(base64)}`; +} + +function resolveImagePayloadObject( + value: unknown, + maxBytes: number, +): ImagePreviewResult | null { + if (!value || typeof value !== "object") { + return null; + } + const payload = value as PreviewImagePayload; + if (payload.type !== "image" || typeof payload.data !== "string") { + return null; + } + const mime = typeof payload.mime === "string" ? payload.mime : "image/jpeg"; + const estimatedBytes = estimateBase64Bytes(payload.data); + if (estimatedBytes > maxBytes) { + return { kind: "too_large", estimatedBytes }; + } + return { + kind: "ready", + src: toDataUrlFromBase64(payload.data, mime), + }; +} + +export function resolveImagePreview( + value: unknown, + maxBytes = MAX_IMAGE_PREVIEW_BYTES, +): ImagePreviewResult | null { + const payloadPreview = resolveImagePayloadObject(value, maxBytes); + if (payloadPreview) { + return payloadPreview; + } + + if (typeof value !== "string") { + return null; + } + const trimmed = value.trim(); + if (!trimmed) { + return null; + } + if (trimmed.startsWith("http://") || trimmed.startsWith("https://")) { + return { kind: "ready", src: trimmed }; + } + if (trimmed.startsWith("data:image/")) { + const marker = "base64,"; + const markerIdx = trimmed.indexOf(marker); + if (markerIdx < 0) { + return { kind: "ready", src: trimmed }; + } + const encoded = trimmed.slice(markerIdx + marker.length); + const estimatedBytes = estimateBase64Bytes(encoded); + if (estimatedBytes > maxBytes) { + return { kind: "too_large", estimatedBytes }; + } + return { kind: "ready", src: trimmed }; + } + if (isLikelyRawBase64Image(trimmed)) { + const estimatedBytes = estimateBase64Bytes(trimmed); + if (estimatedBytes > maxBytes) { + return { kind: "too_large", estimatedBytes }; + } + const mime = inferMimeFromBase64(trimmed) ?? "image/png"; + return { kind: "ready", src: toDataUrlFromBase64(trimmed, mime) }; + } + return null; +} + +export function isLikelyImageValue(value: unknown): boolean { + return resolveImagePreview(value, Number.POSITIVE_INFINITY) !== null; +} diff --git a/studio/frontend/src/features/recipe-studio/utils/import/parsers/llm-parser.ts b/studio/frontend/src/features/recipe-studio/utils/import/parsers/llm-parser.ts index 9007e4e7f6..fa54e76a22 100644 --- a/studio/frontend/src/features/recipe-studio/utils/import/parsers/llm-parser.ts +++ b/studio/frontend/src/features/recipe-studio/utils/import/parsers/llm-parser.ts @@ -44,6 +44,26 @@ export function parseLlm( }) : []; + let imageContext: LlmConfig["image_context"] = { + enabled: false, + // biome-ignore lint/style/useNamingConvention: api schema + column_name: "", + }; + if (Array.isArray(column.multi_modal_context)) { + const first = column.multi_modal_context.find((entry) => isRecord(entry)); + if (first && isRecord(first)) { + const modality = readString(first.modality); + const columnName = readString(first.column_name) ?? ""; + if (modality === "image" && columnName) { + imageContext = { + enabled: true, + // biome-ignore lint/style/useNamingConvention: api schema + column_name: columnName, + }; + } + } + } + return { id, kind: "llm", @@ -63,5 +83,7 @@ export function parseLlm( // biome-ignore lint/style/useNamingConvention: api schema tool_alias: readString(column.tool_alias) ?? "", scores: llmType === "judge" ? scores : undefined, + // biome-ignore lint/style/useNamingConvention: ui schema + image_context: imageContext, }; } diff --git a/studio/frontend/src/features/recipe-studio/utils/payload/build-payload.ts b/studio/frontend/src/features/recipe-studio/utils/payload/build-payload.ts index 385e214b4c..93432be0c0 100644 --- a/studio/frontend/src/features/recipe-studio/utils/payload/build-payload.ts +++ b/studio/frontend/src/features/recipe-studio/utils/payload/build-payload.ts @@ -42,6 +42,7 @@ import { validateTimedeltaConfigs, validateUsedProviders, } from "./validate"; +import { isLikelyImageValue } from "../image-preview"; function pushUniqueJson( label: string, @@ -110,6 +111,30 @@ export function buildRecipePayload( continue; } if (config.kind === "llm") { + if (config.image_context?.enabled) { + const imageContext = config.image_context; + const columnName = imageContext.column_name.trim(); + if (columnName) { + if (firstSeed?.seed_columns && firstSeed.seed_columns.length > 0) { + if (!firstSeed.seed_columns.includes(columnName)) { + errors.push( + `LLM ${config.name}: image context column '${columnName}' not found in seed columns.`, + ); + } + } + const previewRows = firstSeed?.seed_preview_rows ?? []; + if (previewRows.length > 0) { + const hasImageLikeValue = previewRows.some((row) => + isLikelyImageValue(row[columnName]), + ); + if (!hasImageLikeValue) { + errors.push( + `LLM ${config.name}: image context column '${columnName}' has no image-like values in preview rows.`, + ); + } + } + } + } columns.push(buildLlmColumn(config, errors)); for (const provider of config.mcp_providers ?? []) { const builtProvider = buildLlmMcpProvider(provider, errors); diff --git a/studio/frontend/src/features/recipe-studio/utils/payload/builders-llm.ts b/studio/frontend/src/features/recipe-studio/utils/payload/builders-llm.ts index 5c42b2fcd1..8556d83157 100644 --- a/studio/frontend/src/features/recipe-studio/utils/payload/builders-llm.ts +++ b/studio/frontend/src/features/recipe-studio/utils/payload/builders-llm.ts @@ -1,5 +1,27 @@ import type { LlmConfig, LlmMcpProviderConfig, LlmToolConfig } from "../../types"; +function buildImageContext( + config: LlmConfig, + errors: string[], +): Array> | undefined { + const imageContext = config.image_context; + if (!imageContext?.enabled) { + return undefined; + } + const columnName = imageContext.column_name.trim(); + if (!columnName) { + errors.push(`LLM ${config.name}: image context column is required.`); + return undefined; + } + return [ + { + modality: "image", + // biome-ignore lint/style/useNamingConvention: api schema + column_name: columnName, + }, + ]; +} + export function buildLlmColumn( config: LlmConfig, errors: string[], @@ -14,6 +36,8 @@ export function buildLlmColumn( // biome-ignore lint/style/useNamingConvention: api schema system_prompt: config.system_prompt || undefined, // biome-ignore lint/style/useNamingConvention: api schema + multi_modal_context: buildImageContext(config, errors), + // biome-ignore lint/style/useNamingConvention: api schema tool_alias: toolAlias || undefined, }; diff --git a/studio/frontend/src/features/recipe-studio/utils/validation.ts b/studio/frontend/src/features/recipe-studio/utils/validation.ts index 47322f84a7..d0a683aa42 100644 --- a/studio/frontend/src/features/recipe-studio/utils/validation.ts +++ b/studio/frontend/src/features/recipe-studio/utils/validation.ts @@ -173,6 +173,11 @@ export function getConfigErrors(config: NodeConfig | null): string[] { } } } + if (config.image_context?.enabled) { + if (!config.image_context.column_name.trim()) { + errors.push("Image context column is required."); + } + } } if (config.kind === "expression") { if (!config.expr.trim()) {