diff --git a/studio/backend/core/data_recipe/jobs/manager.py b/studio/backend/core/data_recipe/jobs/manager.py
index 2a8b1fe44d..de367dcce6 100644
--- a/studio/backend/core/data_recipe/jobs/manager.py
+++ b/studio/backend/core/data_recipe/jobs/manager.py
@@ -13,7 +13,7 @@ from typing import Any
import multiprocessing as mp
-from ..jsonable import to_jsonable
+from ..jsonable import to_preview_jsonable
from .constants import (
EVENT_JOB_CANCELLING,
EVENT_JOB_CANCELLED,
@@ -299,7 +299,7 @@ class JobManager:
dataframe = dataframe.drop(columns=[helper_col])
rows = dataframe.to_dict(orient="records")
- return {"dataset": to_jsonable(rows), "total": total}
+ return {"dataset": to_preview_jsonable(rows), "total": total}
@staticmethod
def _load_dataset_page_with_data_designer(
@@ -313,7 +313,7 @@ class JobManager:
dataframe = read_parquet_dataset(parquet_dir)
total = int(len(dataframe.index))
rows = dataframe.iloc[offset:offset + limit].to_dict(orient="records")
- return {"dataset": to_jsonable(rows), "total": total}
+ return {"dataset": to_preview_jsonable(rows), "total": total}
def subscribe(self, job_id: str, *, after_seq: int | None = None) -> Subscription | None:
"""SSE subscribe: get replay buffer + live events stream."""
diff --git a/studio/backend/core/data_recipe/jobs/worker.py b/studio/backend/core/data_recipe/jobs/worker.py
index 8c0996b140..d8f744868c 100644
--- a/studio/backend/core/data_recipe/jobs/worker.py
+++ b/studio/backend/core/data_recipe/jobs/worker.py
@@ -7,7 +7,7 @@ import traceback
from pathlib import Path
from typing import Any
-from ..jsonable import to_jsonable
+from ..jsonable import to_jsonable, to_preview_jsonable
from .constants import EVENT_JOB_COMPLETED, EVENT_JOB_ERROR, EVENT_JOB_STARTED
from ..service import build_config_builder, create_data_designer
@@ -84,7 +84,7 @@ def run_job_process(
dataset = (
[]
if results.dataset is None
- else to_jsonable(results.dataset.to_dict(orient="records"))
+ else to_preview_jsonable(results.dataset.to_dict(orient="records"))
)
processor_artifacts = (
None
diff --git a/studio/backend/core/data_recipe/jsonable.py b/studio/backend/core/data_recipe/jsonable.py
index aa6e1d6b2e..a6828b2274 100644
--- a/studio/backend/core/data_recipe/jsonable.py
+++ b/studio/backend/core/data_recipe/jsonable.py
@@ -1,5 +1,7 @@
from __future__ import annotations
+import base64
+import io
from typing import Any
@@ -28,3 +30,41 @@ def to_jsonable(value: Any) -> Any:
return value
return value
+
+
+def _to_preview_image_payload(value: Any) -> dict[str, Any] | None:
+ try:
+ from PIL.Image import Image as PILImage # type: ignore
+ except ImportError: # pragma: no cover
+ return None
+
+ if not isinstance(value, PILImage):
+ return None
+
+ buffer = io.BytesIO()
+ value.convert("RGB").save(buffer, format="JPEG", quality=85)
+ return {
+ "type": "image",
+ "mime": "image/jpeg",
+ "width": value.width,
+ "height": value.height,
+ "data": base64.b64encode(buffer.getvalue()).decode("ascii"),
+ }
+
+
+def to_preview_jsonable(value: Any) -> Any:
+ """Convert values into JSON-safe preview values, including PIL images."""
+ image_payload = _to_preview_image_payload(value)
+ if image_payload is not None:
+ return image_payload
+
+ converted = to_jsonable(value)
+ if converted is None or isinstance(converted, (str, int, float, bool)):
+ return converted
+ if isinstance(converted, dict):
+ return {str(k): to_preview_jsonable(v) for k, v in converted.items()}
+ if isinstance(converted, (list, tuple, set)):
+ return [to_preview_jsonable(v) for v in converted]
+ if isinstance(converted, (bytes, bytearray)):
+ return base64.b64encode(bytes(converted)).decode("ascii")
+ return str(converted)
diff --git a/studio/backend/routes/data_recipe/seed.py b/studio/backend/routes/data_recipe/seed.py
index eb02ab2bbf..1b8df3bc9b 100644
--- a/studio/backend/routes/data_recipe/seed.py
+++ b/studio/backend/routes/data_recipe/seed.py
@@ -10,6 +10,7 @@ from typing import Any
from uuid import uuid4
from fastapi import APIRouter, HTTPException
+from core.data_recipe.jsonable import to_preview_jsonable
from models.data_recipe import (
SeedInspectRequest,
@@ -26,13 +27,7 @@ SEED_UPLOAD_DIR = Path.home() / ".cache" / "unsloth" / "data-recipe" / "seed-upl
def _serialize_preview_value(value: Any) -> Any:
- if value is None or isinstance(value, (str, int, float, bool)):
- return value
- if isinstance(value, dict):
- return {str(key): _serialize_preview_value(item) for key, item in value.items()}
- if isinstance(value, (list, tuple)):
- return [_serialize_preview_value(item) for item in value]
- return str(value)
+ return to_preview_jsonable(value)
def _serialize_preview_rows(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
@@ -63,10 +58,10 @@ def _list_hf_data_files(*, dataset_name: str, token: str | None) -> list[str]:
return []
-def _select_best_file(data_files: list[str]) -> str | None:
+def _select_best_file(data_files: list[str], split: str = DEFAULT_SPLIT) -> str | None:
if not data_files:
return None
- split_lower = DEFAULT_SPLIT
+ split_lower = split.lower()
def score(path: str) -> tuple[int, int]:
name = path.lower()
@@ -85,8 +80,8 @@ def _select_best_file(data_files: list[str]) -> str | None:
return sorted(data_files, key=score)[0]
-def _resolve_seed_hf_path(dataset_name: str, data_files: list[str]) -> str | None:
- selected = _select_best_file(data_files)
+def _resolve_seed_hf_path(dataset_name: str, data_files: list[str], split: str = DEFAULT_SPLIT) -> str | None:
+ selected = _select_best_file(data_files, split)
if not selected:
return None
@@ -196,7 +191,7 @@ def inspect_seed_dataset(payload: SeedInspectRequest) -> SeedInspectResponse:
except ImportError as exc:
raise HTTPException(status_code=500, detail=f"seed inspect dependencies unavailable: {exc}") from exc
- split = DEFAULT_SPLIT
+ split = _normalize_optional_text(payload.split) or DEFAULT_SPLIT
subset = _normalize_optional_text(payload.subset)
token = _normalize_optional_text(payload.hf_token)
preview_size = int(payload.preview_size)
@@ -204,12 +199,12 @@ def inspect_seed_dataset(payload: SeedInspectRequest) -> SeedInspectResponse:
preview_rows: list[dict[str, Any]] = []
data_files = _list_hf_data_files(dataset_name=dataset_name, token=token)
- selected_file = _select_best_file(data_files)
+ selected_file = _select_best_file(data_files, split)
if selected_file:
try:
single_file_kwargs = _build_stream_load_kwargs(
dataset_name=dataset_name,
- split=DEFAULT_SPLIT,
+ split=split,
subset=subset,
token=token,
data_file=selected_file,
@@ -246,7 +241,7 @@ def inspect_seed_dataset(payload: SeedInspectRequest) -> SeedInspectResponse:
if not data_files:
resolved_path = f"datasets/{dataset_name}/**/*.parquet"
else:
- resolved_path = _resolve_seed_hf_path(dataset_name, data_files)
+ resolved_path = _resolve_seed_hf_path(dataset_name, data_files, split)
if not resolved_path:
raise HTTPException(status_code=422, detail="unable to resolve seed dataset path")
@@ -255,7 +250,7 @@ def inspect_seed_dataset(payload: SeedInspectRequest) -> SeedInspectResponse:
resolved_path=resolved_path,
columns=columns,
preview_rows=preview_rows,
- split=None,
+ split=split,
subset=subset,
)
diff --git a/studio/frontend/src/features/recipe-studio/components/executions/execution-data-tab.tsx b/studio/frontend/src/features/recipe-studio/components/executions/execution-data-tab.tsx
index 56ca1d5057..e68a047c85 100644
--- a/studio/frontend/src/features/recipe-studio/components/executions/execution-data-tab.tsx
+++ b/studio/frontend/src/features/recipe-studio/components/executions/execution-data-tab.tsx
@@ -10,6 +10,7 @@ import {
DropdownMenuTrigger,
} from "@/components/ui/dropdown-menu";
import { cn } from "@/lib/utils";
+import { resolveImagePreview } from "../../utils/image-preview";
import { isExecutionInProgress } from "../../executions/execution-helpers";
import type { RecipeExecutionRecord } from "../../execution-types";
import { formatCellValue, isExpandableCellValue } from "./executions-view-helpers";
@@ -130,6 +131,7 @@ export function ExecutionDataTab({
data={datasetRowsForTable}
getRowClassName={(row, _rowIndex, rowId) => {
const canExpand = visibleDatasetColumnNames.some((columnName) =>
+ !resolveImagePreview(row[columnName]) &&
isExpandableCellValue(formatCellValue(row[columnName])),
);
if (!canExpand) {
@@ -142,6 +144,7 @@ export function ExecutionDataTab({
}}
onRowClick={(row, _rowIndex, rowId) => {
const canExpand = visibleDatasetColumnNames.some((columnName) =>
+ !resolveImagePreview(row[columnName]) &&
isExpandableCellValue(formatCellValue(row[columnName])),
);
if (!canExpand || !selectedExecutionIdSafe) {
diff --git a/studio/frontend/src/features/recipe-studio/components/executions/executions-view.tsx b/studio/frontend/src/features/recipe-studio/components/executions/executions-view.tsx
index 4deeb92982..6ebfa6af5b 100644
--- a/studio/frontend/src/features/recipe-studio/components/executions/executions-view.tsx
+++ b/studio/frontend/src/features/recipe-studio/components/executions/executions-view.tsx
@@ -10,6 +10,7 @@ import { Button } from "@/components/ui/button";
import { Progress } from "@/components/ui/progress";
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
import { cn } from "@/lib/utils";
+import { resolveImagePreview } from "../../utils/image-preview";
import type {
RecipeExecutionRecord,
} from "../../execution-types";
@@ -119,9 +120,32 @@ export function ExecutionsView({
header: name,
cell: ({ getValue, row }) => {
const rawValue = getValue();
+ const imagePreview = resolveImagePreview(rawValue);
+ if (imagePreview?.kind === "ready") {
+ return (
+
+

+
+ );
+ }
+ if (imagePreview?.kind === "too_large") {
+ return (
+
+
+ Image too large to preview
+
+
+ );
+ }
const value = formatCellValue(rawValue);
const rowExpanded = Boolean(expandedDatasetRows[row.id]);
const rowHasExpandableCell = visibleDatasetColumnNames.some((columnName) =>
+ !resolveImagePreview(row.original[columnName]) &&
isExpandableCellValue(formatCellValue(row.original[columnName])),
);
const showTruncated = rowHasExpandableCell && !rowExpanded;
diff --git a/studio/frontend/src/features/recipe-studio/dialogs/llm/general-tab.tsx b/studio/frontend/src/features/recipe-studio/dialogs/llm/general-tab.tsx
index 1fbf267d47..547e8745ce 100644
--- a/studio/frontend/src/features/recipe-studio/dialogs/llm/general-tab.tsx
+++ b/studio/frontend/src/features/recipe-studio/dialogs/llm/general-tab.tsx
@@ -6,6 +6,7 @@ import {
ComboboxItem,
ComboboxList,
} from "@/components/ui/combobox";
+import { Switch } from "@/components/ui/switch";
import {
Select,
SelectContent,
@@ -16,6 +17,7 @@ import {
import { Textarea } from "@/components/ui/textarea";
import { type ReactElement, type RefObject, useMemo } from "react";
import { useRecipeStudioStore } from "../../stores/recipe-studio";
+import { isLikelyImageValue } from "../../utils/image-preview";
import type { LlmConfig } from "../../types";
import { findInvalidJinjaReferences } from "../../utils/refs";
import { getAvailableVariables } from "../../utils/variables";
@@ -85,6 +87,39 @@ export function LlmGeneralTab({
.slice(0, 3)
.map((ref) => `{{ ${ref} }}`)
.join(", ");
+ const seedConfig = useMemo(
+ () => Object.values(configs).find((item) => item.kind === "seed"),
+ [configs],
+ );
+ const seedColumns = seedConfig?.seed_columns ?? [];
+ const seedPreviewRows = seedConfig?.seed_preview_rows ?? [];
+ const imageColumnOptions = useMemo(() => {
+ if (seedColumns.length === 0) {
+ return [];
+ }
+ const detected = seedColumns.filter((columnName) => {
+ const lower = columnName.toLowerCase();
+ if (
+ lower.includes("image") ||
+ lower.includes("img") ||
+ lower.includes("photo") ||
+ lower.includes("picture") ||
+ lower.includes("base64") ||
+ lower.includes("url")
+ ) {
+ return true;
+ }
+ return seedPreviewRows.some((row) => isLikelyImageValue(row[columnName]));
+ });
+ return detected.length > 0 ? detected : seedColumns;
+ }, [seedColumns, seedPreviewRows]);
+ const imageContext = config.image_context ?? {
+ enabled: false,
+ // biome-ignore lint/style/useNamingConvention: api schema
+ column_name: "",
+ };
+ const imageContextToggleId = `${config.id}-image-context-enabled`;
+ const imageContextColumnId = `${config.id}-image-context-column`;
return (
@@ -185,6 +220,71 @@ export function LlmGeneralTab({
)}
+
+
+
+
+ {imageColumnOptions.length > 0 && (
+
+ Suggested image columns: {imageColumnOptions.join(", ")}
+
+ )}
+
+
{
+ onUpdate({
+ image_context: {
+ ...imageContext,
+ enabled: checked,
+ // biome-ignore lint/style/useNamingConvention: api schema
+ column_name:
+ checked && !imageContext.column_name
+ ? (imageColumnOptions[0] ?? "")
+ : imageContext.column_name,
+ },
+ });
+ }}
+ />
+
+ {imageContext.enabled && (
+
+
+
+
+ )}
+
{config.llm_type === "structured" && (
new Set(selectedSeedDropColumns),
[selectedSeedDropColumns],
);
+ const rowHasExpandableText = useCallback(
+ (row: Record): boolean =>
+ previewColumns.some((columnName) => {
+ if (resolveImagePreview(row[columnName])) {
+ return false;
+ }
+ return isExpandablePreviewValue(stringifyCell(row[columnName]));
+ }),
+ [previewColumns],
+ );
return (
@@ -778,15 +790,11 @@ export function SeedDialog({ config, onUpdate, open }: SeedDialogProps): ReactEl
- isExpandablePreviewValue(stringifyCell(row[col])),
- ) && "cursor-pointer hover:bg-primary/[0.06]",
+ rowHasExpandableText(row) && "cursor-pointer hover:bg-primary/[0.06]",
expandedPreviewRows[rowIdx] && "bg-primary/[0.05]",
)}
onClick={() => {
- const canExpand = previewColumns.some((col) =>
- isExpandablePreviewValue(stringifyCell(row[col])),
- );
+ const canExpand = rowHasExpandableText(row);
if (!canExpand) {
return;
}
@@ -802,10 +810,22 @@ export function SeedDialog({ config, onUpdate, open }: SeedDialogProps): ReactEl
className="max-w-[260px] whitespace-pre-wrap break-words text-xs"
>
{(() => {
+ const imagePreview = resolveImagePreview(row[col]);
+ if (imagePreview?.kind === "ready") {
+ return (
+
+ );
+ }
+ if (imagePreview?.kind === "too_large") {
+ return "Image too large to preview";
+ }
const value = stringifyCell(row[col]);
- const rowHasExpandableCell = previewColumns.some((columnName) =>
- isExpandablePreviewValue(stringifyCell(row[columnName])),
- );
+ const rowHasExpandableCell = rowHasExpandableText(row);
const rowExpanded = Boolean(expandedPreviewRows[rowIdx]);
return rowHasExpandableCell && !rowExpanded
? truncatePreviewValue(value)
diff --git a/studio/frontend/src/features/recipe-studio/types/index.ts b/studio/frontend/src/features/recipe-studio/types/index.ts
index 7d36de28cd..d12002edcc 100644
--- a/studio/frontend/src/features/recipe-studio/types/index.ts
+++ b/studio/frontend/src/features/recipe-studio/types/index.ts
@@ -152,6 +152,12 @@ export type LlmToolConfig = {
timeout_sec?: string;
};
+export type LlmImageContextConfig = {
+ enabled: boolean;
+ // biome-ignore lint/style/useNamingConvention: api schema
+ column_name: string;
+};
+
export type LlmConfig = {
id: string;
kind: "llm";
@@ -175,6 +181,9 @@ export type LlmConfig = {
// biome-ignore lint/style/useNamingConvention: ui schema
mcp_providers?: LlmMcpProviderConfig[];
scores?: Score[];
+ // ui-only, serialized into multi_modal_context for DataDesigner
+ // biome-ignore lint/style/useNamingConvention: ui schema
+ image_context?: LlmImageContextConfig;
};
export type ModelProviderConfig = {
diff --git a/studio/frontend/src/features/recipe-studio/utils/config-factories.ts b/studio/frontend/src/features/recipe-studio/utils/config-factories.ts
index f92589b213..9b711c91f5 100644
--- a/studio/frontend/src/features/recipe-studio/utils/config-factories.ts
+++ b/studio/frontend/src/features/recipe-studio/utils/config-factories.ts
@@ -204,6 +204,12 @@ export function makeLlmConfig(
tool_configs: [],
// biome-ignore lint/style/useNamingConvention: ui schema
mcp_providers: [],
+ // biome-ignore lint/style/useNamingConvention: ui schema
+ image_context: {
+ enabled: false,
+ // biome-ignore lint/style/useNamingConvention: api schema
+ column_name: "",
+ },
scores:
llmType === "judge"
? [
diff --git a/studio/frontend/src/features/recipe-studio/utils/image-preview.ts b/studio/frontend/src/features/recipe-studio/utils/image-preview.ts
new file mode 100644
index 0000000000..61099b8f4a
--- /dev/null
+++ b/studio/frontend/src/features/recipe-studio/utils/image-preview.ts
@@ -0,0 +1,126 @@
+export const MAX_IMAGE_PREVIEW_BYTES = 200 * 1024;
+
+type PreviewImagePayload = {
+ type?: unknown;
+ mime?: unknown;
+ data?: unknown;
+};
+
+export type ImagePreviewResult =
+ | { kind: "ready"; src: string }
+ | { kind: "too_large"; estimatedBytes: number };
+
+function normalizeBase64(value: string): string {
+ return value.replace(/\s+/g, "");
+}
+
+function estimateBase64Bytes(base64: string): number {
+ const normalized = normalizeBase64(base64);
+ const padding = normalized.endsWith("==")
+ ? 2
+ : normalized.endsWith("=")
+ ? 1
+ : 0;
+ return Math.max(0, Math.floor((normalized.length * 3) / 4) - padding);
+}
+
+function inferMimeFromBase64(base64: string): string | null {
+ const normalized = normalizeBase64(base64);
+ if (normalized.startsWith("iVBORw0KGgo")) {
+ return "image/png";
+ }
+ if (normalized.startsWith("/9j/")) {
+ return "image/jpeg";
+ }
+ if (normalized.startsWith("R0lGOD")) {
+ return "image/gif";
+ }
+ if (normalized.startsWith("UklGR")) {
+ return "image/webp";
+ }
+ return null;
+}
+
+function isLikelyRawBase64Image(value: string): boolean {
+ const normalized = normalizeBase64(value);
+ if (normalized.length < 64) {
+ return false;
+ }
+ if (!/^[A-Za-z0-9+/=]+$/.test(normalized)) {
+ return false;
+ }
+ return inferMimeFromBase64(normalized) !== null;
+}
+
+function toDataUrlFromBase64(base64: string, mime: string): string {
+ return `data:${mime};base64,${normalizeBase64(base64)}`;
+}
+
+function resolveImagePayloadObject(
+ value: unknown,
+ maxBytes: number,
+): ImagePreviewResult | null {
+ if (!value || typeof value !== "object") {
+ return null;
+ }
+ const payload = value as PreviewImagePayload;
+ if (payload.type !== "image" || typeof payload.data !== "string") {
+ return null;
+ }
+ const mime = typeof payload.mime === "string" ? payload.mime : "image/jpeg";
+ const estimatedBytes = estimateBase64Bytes(payload.data);
+ if (estimatedBytes > maxBytes) {
+ return { kind: "too_large", estimatedBytes };
+ }
+ return {
+ kind: "ready",
+ src: toDataUrlFromBase64(payload.data, mime),
+ };
+}
+
+export function resolveImagePreview(
+ value: unknown,
+ maxBytes = MAX_IMAGE_PREVIEW_BYTES,
+): ImagePreviewResult | null {
+ const payloadPreview = resolveImagePayloadObject(value, maxBytes);
+ if (payloadPreview) {
+ return payloadPreview;
+ }
+
+ if (typeof value !== "string") {
+ return null;
+ }
+ const trimmed = value.trim();
+ if (!trimmed) {
+ return null;
+ }
+ if (trimmed.startsWith("http://") || trimmed.startsWith("https://")) {
+ return { kind: "ready", src: trimmed };
+ }
+ if (trimmed.startsWith("data:image/")) {
+ const marker = "base64,";
+ const markerIdx = trimmed.indexOf(marker);
+ if (markerIdx < 0) {
+ return { kind: "ready", src: trimmed };
+ }
+ const encoded = trimmed.slice(markerIdx + marker.length);
+ const estimatedBytes = estimateBase64Bytes(encoded);
+ if (estimatedBytes > maxBytes) {
+ return { kind: "too_large", estimatedBytes };
+ }
+ return { kind: "ready", src: trimmed };
+ }
+ if (isLikelyRawBase64Image(trimmed)) {
+ const estimatedBytes = estimateBase64Bytes(trimmed);
+ if (estimatedBytes > maxBytes) {
+ return { kind: "too_large", estimatedBytes };
+ }
+ const mime = inferMimeFromBase64(trimmed) ?? "image/png";
+ return { kind: "ready", src: toDataUrlFromBase64(trimmed, mime) };
+ }
+ return null;
+}
+
+export function isLikelyImageValue(value: unknown): boolean {
+ return resolveImagePreview(value, Number.POSITIVE_INFINITY) !== null;
+}
diff --git a/studio/frontend/src/features/recipe-studio/utils/import/parsers/llm-parser.ts b/studio/frontend/src/features/recipe-studio/utils/import/parsers/llm-parser.ts
index 9007e4e7f6..fa54e76a22 100644
--- a/studio/frontend/src/features/recipe-studio/utils/import/parsers/llm-parser.ts
+++ b/studio/frontend/src/features/recipe-studio/utils/import/parsers/llm-parser.ts
@@ -44,6 +44,26 @@ export function parseLlm(
})
: [];
+ let imageContext: LlmConfig["image_context"] = {
+ enabled: false,
+ // biome-ignore lint/style/useNamingConvention: api schema
+ column_name: "",
+ };
+ if (Array.isArray(column.multi_modal_context)) {
+ const first = column.multi_modal_context.find((entry) => isRecord(entry));
+ if (first && isRecord(first)) {
+ const modality = readString(first.modality);
+ const columnName = readString(first.column_name) ?? "";
+ if (modality === "image" && columnName) {
+ imageContext = {
+ enabled: true,
+ // biome-ignore lint/style/useNamingConvention: api schema
+ column_name: columnName,
+ };
+ }
+ }
+ }
+
return {
id,
kind: "llm",
@@ -63,5 +83,7 @@ export function parseLlm(
// biome-ignore lint/style/useNamingConvention: api schema
tool_alias: readString(column.tool_alias) ?? "",
scores: llmType === "judge" ? scores : undefined,
+ // biome-ignore lint/style/useNamingConvention: ui schema
+ image_context: imageContext,
};
}
diff --git a/studio/frontend/src/features/recipe-studio/utils/payload/build-payload.ts b/studio/frontend/src/features/recipe-studio/utils/payload/build-payload.ts
index 385e214b4c..93432be0c0 100644
--- a/studio/frontend/src/features/recipe-studio/utils/payload/build-payload.ts
+++ b/studio/frontend/src/features/recipe-studio/utils/payload/build-payload.ts
@@ -42,6 +42,7 @@ import {
validateTimedeltaConfigs,
validateUsedProviders,
} from "./validate";
+import { isLikelyImageValue } from "../image-preview";
function pushUniqueJson(
label: string,
@@ -110,6 +111,30 @@ export function buildRecipePayload(
continue;
}
if (config.kind === "llm") {
+ if (config.image_context?.enabled) {
+ const imageContext = config.image_context;
+ const columnName = imageContext.column_name.trim();
+ if (columnName) {
+ if (firstSeed?.seed_columns && firstSeed.seed_columns.length > 0) {
+ if (!firstSeed.seed_columns.includes(columnName)) {
+ errors.push(
+ `LLM ${config.name}: image context column '${columnName}' not found in seed columns.`,
+ );
+ }
+ }
+ const previewRows = firstSeed?.seed_preview_rows ?? [];
+ if (previewRows.length > 0) {
+ const hasImageLikeValue = previewRows.some((row) =>
+ isLikelyImageValue(row[columnName]),
+ );
+ if (!hasImageLikeValue) {
+ errors.push(
+ `LLM ${config.name}: image context column '${columnName}' has no image-like values in preview rows.`,
+ );
+ }
+ }
+ }
+ }
columns.push(buildLlmColumn(config, errors));
for (const provider of config.mcp_providers ?? []) {
const builtProvider = buildLlmMcpProvider(provider, errors);
diff --git a/studio/frontend/src/features/recipe-studio/utils/payload/builders-llm.ts b/studio/frontend/src/features/recipe-studio/utils/payload/builders-llm.ts
index 5c42b2fcd1..8556d83157 100644
--- a/studio/frontend/src/features/recipe-studio/utils/payload/builders-llm.ts
+++ b/studio/frontend/src/features/recipe-studio/utils/payload/builders-llm.ts
@@ -1,5 +1,27 @@
import type { LlmConfig, LlmMcpProviderConfig, LlmToolConfig } from "../../types";
+function buildImageContext(
+ config: LlmConfig,
+ errors: string[],
+): Array> | undefined {
+ const imageContext = config.image_context;
+ if (!imageContext?.enabled) {
+ return undefined;
+ }
+ const columnName = imageContext.column_name.trim();
+ if (!columnName) {
+ errors.push(`LLM ${config.name}: image context column is required.`);
+ return undefined;
+ }
+ return [
+ {
+ modality: "image",
+ // biome-ignore lint/style/useNamingConvention: api schema
+ column_name: columnName,
+ },
+ ];
+}
+
export function buildLlmColumn(
config: LlmConfig,
errors: string[],
@@ -14,6 +36,8 @@ export function buildLlmColumn(
// biome-ignore lint/style/useNamingConvention: api schema
system_prompt: config.system_prompt || undefined,
// biome-ignore lint/style/useNamingConvention: api schema
+ multi_modal_context: buildImageContext(config, errors),
+ // biome-ignore lint/style/useNamingConvention: api schema
tool_alias: toolAlias || undefined,
};
diff --git a/studio/frontend/src/features/recipe-studio/utils/validation.ts b/studio/frontend/src/features/recipe-studio/utils/validation.ts
index 47322f84a7..d0a683aa42 100644
--- a/studio/frontend/src/features/recipe-studio/utils/validation.ts
+++ b/studio/frontend/src/features/recipe-studio/utils/validation.ts
@@ -173,6 +173,11 @@ export function getConfigErrors(config: NodeConfig | null): string[] {
}
}
}
+ if (config.image_context?.enabled) {
+ if (!config.image_context.column_name.trim()) {
+ errors.push("Image context column is required.");
+ }
+ }
}
if (config.kind === "expression") {
if (!config.expr.trim()) {