ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(recipe-studio): add image preview support for dataset and LLM configurations p1

Browse source
This commit is contained in:
Shine1i 2026-03-01 10:57:51 +01:00
commit c3c65cded8
15 changed files with 432 additions and 33 deletions
Download patch file Download diff file Expand all files Collapse all files

6
studio/backend/core/data_recipe/jobs/manager.py
View file

@ -13,7 +13,7 @@ from typing import Any
import multiprocessing as mp
from ..jsonable import to_jsonable
from ..jsonable import to_preview_jsonable
from .constants import (
EVENT_JOB_CANCELLING,
EVENT_JOB_CANCELLED,
@ -299,7 +299,7 @@ class JobManager:
dataframe = dataframe.drop(columns=[helper_col])
rows = dataframe.to_dict(orient="records")
return {"dataset": to_jsonable(rows), "total": total}
return {"dataset": to_preview_jsonable(rows), "total": total}
@staticmethod
def _load_dataset_page_with_data_designer(
@ -313,7 +313,7 @@ class JobManager:
dataframe = read_parquet_dataset(parquet_dir)
total = int(len(dataframe.index))
rows = dataframe.iloc[offset:offset + limit].to_dict(orient="records")
return {"dataset": to_jsonable(rows), "total": total}
return {"dataset": to_preview_jsonable(rows), "total": total}
def subscribe(self, job_id: str, *, after_seq: int | None = None) -> Subscription | None:
"""SSE subscribe: get replay buffer + live events stream."""

4
studio/backend/core/data_recipe/jobs/worker.py
View file

@ -7,7 +7,7 @@ import traceback
from pathlib import Path
from typing import Any
from ..jsonable import to_jsonable
from ..jsonable import to_jsonable, to_preview_jsonable
from .constants import EVENT_JOB_COMPLETED, EVENT_JOB_ERROR, EVENT_JOB_STARTED
from ..service import build_config_builder, create_data_designer
@ -84,7 +84,7 @@ def run_job_process(
dataset = (
[]
if results.dataset is None
else to_jsonable(results.dataset.to_dict(orient="records"))
else to_preview_jsonable(results.dataset.to_dict(orient="records"))
)
processor_artifacts = (
None

40
studio/backend/core/data_recipe/jsonable.py
View file

@ -1,5 +1,7 @@
from __future__ import annotations
import base64
import io
from typing import Any
@ -28,3 +30,41 @@ def to_jsonable(value: Any) -> Any:
return value
return value
def _to_preview_image_payload(value: Any) -> dict[str, Any] | None:
try:
from PIL.Image import Image as PILImage # type: ignore
except ImportError: # pragma: no cover
return None
if not isinstance(value, PILImage):
return None
buffer = io.BytesIO()
value.convert("RGB").save(buffer, format="JPEG", quality=85)
return {
"type": "image",
"mime": "image/jpeg",
"width": value.width,
"height": value.height,
"data": base64.b64encode(buffer.getvalue()).decode("ascii"),
}
def to_preview_jsonable(value: Any) -> Any:
"""Convert values into JSON-safe preview values, including PIL images."""
image_payload = _to_preview_image_payload(value)
if image_payload is not None:
return image_payload
converted = to_jsonable(value)
if converted is None or isinstance(converted, (str, int, float, bool)):
return converted
if isinstance(converted, dict):
return {str(k): to_preview_jsonable(v) for k, v in converted.items()}
if isinstance(converted, (list, tuple, set)):
return [to_preview_jsonable(v) for v in converted]
if isinstance(converted, (bytes, bytearray)):
return base64.b64encode(bytes(converted)).decode("ascii")
return str(converted)

27
studio/backend/routes/data_recipe/seed.py
View file

@ -10,6 +10,7 @@ from typing import Any
from uuid import uuid4
from fastapi import APIRouter, HTTPException
from core.data_recipe.jsonable import to_preview_jsonable
from models.data_recipe import (
SeedInspectRequest,
@ -26,13 +27,7 @@ SEED_UPLOAD_DIR = Path.home() / ".cache" / "unsloth" / "data-recipe" / "seed-upl
def _serialize_preview_value(value: Any) -> Any:
if value is None or isinstance(value, (str, int, float, bool)):
return value
if isinstance(value, dict):
return {str(key): _serialize_preview_value(item) for key, item in value.items()}
if isinstance(value, (list, tuple)):
return [_serialize_preview_value(item) for item in value]
return str(value)
return to_preview_jsonable(value)
def _serialize_preview_rows(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
@ -63,10 +58,10 @@ def _list_hf_data_files(*, dataset_name: str, token: str | None) -> list[str]:
return []
def _select_best_file(data_files: list[str]) -> str | None:
def _select_best_file(data_files: list[str], split: str = DEFAULT_SPLIT) -> str | None:
if not data_files:
return None
split_lower = DEFAULT_SPLIT
split_lower = split.lower()
def score(path: str) -> tuple[int, int]:
name = path.lower()
@ -85,8 +80,8 @@ def _select_best_file(data_files: list[str]) -> str | None:
return sorted(data_files, key=score)[0]
def _resolve_seed_hf_path(dataset_name: str, data_files: list[str]) -> str | None:
selected = _select_best_file(data_files)
def _resolve_seed_hf_path(dataset_name: str, data_files: list[str], split: str = DEFAULT_SPLIT) -> str | None:
selected = _select_best_file(data_files, split)
if not selected:
return None
@ -196,7 +191,7 @@ def inspect_seed_dataset(payload: SeedInspectRequest) -> SeedInspectResponse:
except ImportError as exc:
raise HTTPException(status_code=500, detail=f"seed inspect dependencies unavailable: {exc}") from exc
split = DEFAULT_SPLIT
split = _normalize_optional_text(payload.split) or DEFAULT_SPLIT
subset = _normalize_optional_text(payload.subset)
token = _normalize_optional_text(payload.hf_token)
preview_size = int(payload.preview_size)
@ -204,12 +199,12 @@ def inspect_seed_dataset(payload: SeedInspectRequest) -> SeedInspectResponse:
preview_rows: list[dict[str, Any]] = []
data_files = _list_hf_data_files(dataset_name=dataset_name, token=token)
selected_file = _select_best_file(data_files)
selected_file = _select_best_file(data_files, split)
if selected_file:
try:
single_file_kwargs = _build_stream_load_kwargs(
dataset_name=dataset_name,
split=DEFAULT_SPLIT,
split=split,
subset=subset,
token=token,
data_file=selected_file,
@ -246,7 +241,7 @@ def inspect_seed_dataset(payload: SeedInspectRequest) -> SeedInspectResponse:
if not data_files:
resolved_path = f"datasets/{dataset_name}/**/*.parquet"
else:
resolved_path = _resolve_seed_hf_path(dataset_name, data_files)
resolved_path = _resolve_seed_hf_path(dataset_name, data_files, split)
if not resolved_path:
raise HTTPException(status_code=422, detail="unable to resolve seed dataset path")
@ -255,7 +250,7 @@ def inspect_seed_dataset(payload: SeedInspectRequest) -> SeedInspectResponse:
resolved_path=resolved_path,
columns=columns,
preview_rows=preview_rows,
split=None,
split=split,
subset=subset,
)

3
studio/frontend/src/features/recipe-studio/components/executions/execution-data-tab.tsx
View file

@ -10,6 +10,7 @@ import {
DropdownMenuTrigger,
} from "@/components/ui/dropdown-menu";
import { cn } from "@/lib/utils";
import { resolveImagePreview } from "../../utils/image-preview";
import { isExecutionInProgress } from "../../executions/execution-helpers";
import type { RecipeExecutionRecord } from "../../execution-types";
import { formatCellValue, isExpandableCellValue } from "./executions-view-helpers";
@ -130,6 +131,7 @@ export function ExecutionDataTab({
data={datasetRowsForTable}
getRowClassName={(row, _rowIndex, rowId) => {
const canExpand = visibleDatasetColumnNames.some((columnName) =>
!resolveImagePreview(row[columnName]) &&
isExpandableCellValue(formatCellValue(row[columnName])),
);
if (!canExpand) {
@ -142,6 +144,7 @@ export function ExecutionDataTab({
}}
onRowClick={(row, _rowIndex, rowId) => {
const canExpand = visibleDatasetColumnNames.some((columnName) =>
!resolveImagePreview(row[columnName]) &&
isExpandableCellValue(formatCellValue(row[columnName])),
);
if (!canExpand || !selectedExecutionIdSafe) {

24
studio/frontend/src/features/recipe-studio/components/executions/executions-view.tsx
View file

@ -10,6 +10,7 @@ import { Button } from "@/components/ui/button";
import { Progress } from "@/components/ui/progress";
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
import { cn } from "@/lib/utils";
import { resolveImagePreview } from "../../utils/image-preview";
import type {
RecipeExecutionRecord,
} from "../../execution-types";
@ -119,9 +120,32 @@ export function ExecutionsView({
header: name,
cell: ({ getValue, row }) => {
const rawValue = getValue();
const imagePreview = resolveImagePreview(rawValue);
if (imagePreview?.kind === "ready") {
return (
<div className="max-w-[32rem]">
<img
src={imagePreview.src}
alt={`${name} preview`}
loading="lazy"
className="h-24 w-auto max-w-[260px] rounded-md border border-border/60 bg-muted/20 object-contain"
/>
</div>
);
}
if (imagePreview?.kind === "too_large") {
return (
<div className="max-w-[32rem]">
<p className="text-xs text-muted-foreground">
Image too large to preview
</p>
</div>
);
}
const value = formatCellValue(rawValue);
const rowExpanded = Boolean(expandedDatasetRows[row.id]);
const rowHasExpandableCell = visibleDatasetColumnNames.some((columnName) =>
!resolveImagePreview(row.original[columnName]) &&
isExpandableCellValue(formatCellValue(row.original[columnName])),
);
const showTruncated = rowHasExpandableCell && !rowExpanded;

100
studio/frontend/src/features/recipe-studio/dialogs/llm/general-tab.tsx
View file

@ -6,6 +6,7 @@ import {
ComboboxItem,
ComboboxList,
} from "@/components/ui/combobox";
import { Switch } from "@/components/ui/switch";
import {
Select,
SelectContent,
@ -16,6 +17,7 @@ import {
import { Textarea } from "@/components/ui/textarea";
import { type ReactElement, type RefObject, useMemo } from "react";
import { useRecipeStudioStore } from "../../stores/recipe-studio";
import { isLikelyImageValue } from "../../utils/image-preview";
import type { LlmConfig } from "../../types";
import { findInvalidJinjaReferences } from "../../utils/refs";
import { getAvailableVariables } from "../../utils/variables";
@ -85,6 +87,39 @@ export function LlmGeneralTab({
.slice(0, 3)
.map((ref) => `{{ ${ref} }}`)
.join(", ");
const seedConfig = useMemo(
() => Object.values(configs).find((item) => item.kind === "seed"),
[configs],
);
const seedColumns = seedConfig?.seed_columns ?? [];
const seedPreviewRows = seedConfig?.seed_preview_rows ?? [];
const imageColumnOptions = useMemo(() => {
if (seedColumns.length === 0) {
return [];
}
const detected = seedColumns.filter((columnName) => {
const lower = columnName.toLowerCase();
if (
lower.includes("image") ||
lower.includes("img") ||
lower.includes("photo") ||
lower.includes("picture") ||
lower.includes("base64") ||
lower.includes("url")
) {
return true;
}
return seedPreviewRows.some((row) => isLikelyImageValue(row[columnName]));
});
return detected.length > 0 ? detected : seedColumns;
}, [seedColumns, seedPreviewRows]);
const imageContext = config.image_context ?? {
enabled: false,
// biome-ignore lint/style/useNamingConvention: api schema
column_name: "",
};
const imageContextToggleId = `${config.id}-image-context-enabled`;
const imageContextColumnId = `${config.id}-image-context-column`;
return (
<div className="space-y-4">
@ -185,6 +220,71 @@ export function LlmGeneralTab({
</p>
)}
</div>
<div className="space-y-3 rounded-2xl border border-border/60 px-3 py-3">
<div className="flex items-center justify-between gap-3">
<div>
<FieldLabel
label="Use image context"
htmlFor={imageContextToggleId}
hint="Attach one seed image column to this LLM call."
/>
{imageColumnOptions.length > 0 && (
<p className="text-xs text-muted-foreground">
Suggested image columns: {imageColumnOptions.join(", ")}
</p>
)}
</div>
<Switch
id={imageContextToggleId}
checked={imageContext.enabled}
onCheckedChange={(checked) => {
onUpdate({
image_context: {
...imageContext,
enabled: checked,
// biome-ignore lint/style/useNamingConvention: api schema
column_name:
checked && !imageContext.column_name
? (imageColumnOptions[0] ?? "")
: imageContext.column_name,
},
});
}}
/>
</div>
{imageContext.enabled && (
<div className="grid gap-2">
<FieldLabel
label="Image column"
htmlFor={imageContextColumnId}
hint="Seed column containing image values."
/>
<Select
value={imageContext.column_name || ""}
onValueChange={(value) =>
onUpdate({
image_context: {
...imageContext,
// biome-ignore lint/style/useNamingConvention: api schema
column_name: value,
},
})
}
>
<SelectTrigger className="nodrag w-full" id={imageContextColumnId}>
<SelectValue placeholder="Select image column" />
</SelectTrigger>
<SelectContent>
{imageColumnOptions.map((columnName) => (
<SelectItem key={columnName} value={columnName}>
{columnName}
</SelectItem>
))}
</SelectContent>
</Select>
</div>
)}
</div>
{config.llm_type === "structured" && (
<div className="grid gap-2">
<FieldLabel

44
studio/frontend/src/features/recipe-studio/dialogs/seed/seed-dialog.tsx
View file

@ -40,6 +40,7 @@ import { type ReactElement, useCallback, useEffect, useMemo, useRef, useState }
import { extractText, getDocumentProxy } from "unpdf";
import { cn } from "@/lib/utils";
import { inspectSeedDataset, inspectSeedUpload } from "../../api";
import { resolveImagePreview } from "../../utils/image-preview";
import type {
SeedConfig,
SeedSamplingStrategy,
@ -252,7 +253,8 @@ export function SeedDialog({ config, onUpdate, open }: SeedDialogProps): ReactEl
const response = await inspectSeedDataset({
dataset_name: datasetName,
hf_token: config.hf_token?.trim() || undefined,
subset: undefined,
split: config.hf_split?.trim() || undefined,
subset: config.hf_subset?.trim() || undefined,
preview_size: 10,
});
onUpdate({
@ -262,8 +264,8 @@ export function SeedDialog({ config, onUpdate, open }: SeedDialogProps): ReactEl
response.columns.includes(name),
),
seed_preview_rows: response.preview_rows ?? [],
hf_split: "",
hf_subset: "",
hf_split: response.split ?? "",
hf_subset: response.subset ?? "",
local_file_name: "",
unstructured_file_name: "",
});
@ -393,6 +395,16 @@ export function SeedDialog({ config, onUpdate, open }: SeedDialogProps): ReactEl
() => new Set(selectedSeedDropColumns),
[selectedSeedDropColumns],
);
const rowHasExpandableText = useCallback(
(row: Record<string, unknown>): boolean =>
previewColumns.some((columnName) => {
if (resolveImagePreview(row[columnName])) {
return false;
}
return isExpandablePreviewValue(stringifyCell(row[columnName]));
}),
[previewColumns],
);
return (
<Tabs defaultValue="config" className="w-full">
@ -778,15 +790,11 @@ export function SeedDialog({ config, onUpdate, open }: SeedDialogProps): ReactEl
<TableRow
key={`row-${rowIdx}`}
className={cn(
previewColumns.some((col) =>
isExpandablePreviewValue(stringifyCell(row[col])),
) && "cursor-pointer hover:bg-primary/[0.06]",
rowHasExpandableText(row) && "cursor-pointer hover:bg-primary/[0.06]",
expandedPreviewRows[rowIdx] && "bg-primary/[0.05]",
)}
onClick={() => {
const canExpand = previewColumns.some((col) =>
isExpandablePreviewValue(stringifyCell(row[col])),
);
const canExpand = rowHasExpandableText(row);
if (!canExpand) {
return;
}
@ -802,10 +810,22 @@ export function SeedDialog({ config, onUpdate, open }: SeedDialogProps): ReactEl
className="max-w-[260px] whitespace-pre-wrap break-words text-xs"
>
{(() => {
const imagePreview = resolveImagePreview(row[col]);
if (imagePreview?.kind === "ready") {
return (
<img
src={imagePreview.src}
alt={`${col} preview`}
loading="lazy"
className="h-20 w-auto max-w-[220px] rounded-md border border-border/60 bg-muted/20 object-contain"
/>
);
}
if (imagePreview?.kind === "too_large") {
return "Image too large to preview";
}
const value = stringifyCell(row[col]);
const rowHasExpandableCell = previewColumns.some((columnName) =>
isExpandablePreviewValue(stringifyCell(row[columnName])),
);
const rowHasExpandableCell = rowHasExpandableText(row);
const rowExpanded = Boolean(expandedPreviewRows[rowIdx]);
return rowHasExpandableCell && !rowExpanded
? truncatePreviewValue(value)

9
studio/frontend/src/features/recipe-studio/types/index.ts
View file

@ -152,6 +152,12 @@ export type LlmToolConfig = {
timeout_sec?: string;
};
export type LlmImageContextConfig = {
enabled: boolean;
// biome-ignore lint/style/useNamingConvention: api schema
column_name: string;
};
export type LlmConfig = {
id: string;
kind: "llm";
@ -175,6 +181,9 @@ export type LlmConfig = {
// biome-ignore lint/style/useNamingConvention: ui schema
mcp_providers?: LlmMcpProviderConfig[];
scores?: Score[];
// ui-only, serialized into multi_modal_context for DataDesigner
// biome-ignore lint/style/useNamingConvention: ui schema
image_context?: LlmImageContextConfig;
};
export type ModelProviderConfig = {

6
studio/frontend/src/features/recipe-studio/utils/config-factories.ts
View file

@ -204,6 +204,12 @@ export function makeLlmConfig(
tool_configs: [],
// biome-ignore lint/style/useNamingConvention: ui schema
mcp_providers: [],
// biome-ignore lint/style/useNamingConvention: ui schema
image_context: {
enabled: false,
// biome-ignore lint/style/useNamingConvention: api schema
column_name: "",
},
scores:
llmType === "judge"
? [

126
studio/frontend/src/features/recipe-studio/utils/image-preview.ts Normal file
View file

@ -0,0 +1,126 @@
export const MAX_IMAGE_PREVIEW_BYTES = 200 * 1024;
type PreviewImagePayload = {
type?: unknown;
mime?: unknown;
data?: unknown;
};
export type ImagePreviewResult =
| { kind: "ready"; src: string }
| { kind: "too_large"; estimatedBytes: number };
function normalizeBase64(value: string): string {
return value.replace(/\s+/g, "");
}
function estimateBase64Bytes(base64: string): number {
const normalized = normalizeBase64(base64);
const padding = normalized.endsWith("==")
? 2
: normalized.endsWith("=")
? 1
: 0;
return Math.max(0, Math.floor((normalized.length * 3) / 4) - padding);
}
function inferMimeFromBase64(base64: string): string | null {
const normalized = normalizeBase64(base64);
if (normalized.startsWith("iVBORw0KGgo")) {
return "image/png";
}
if (normalized.startsWith("/9j/")) {
return "image/jpeg";
}
if (normalized.startsWith("R0lGOD")) {
return "image/gif";
}
if (normalized.startsWith("UklGR")) {
return "image/webp";
}
return null;
}
function isLikelyRawBase64Image(value: string): boolean {
const normalized = normalizeBase64(value);
if (normalized.length < 64) {
return false;
}
if (!/^[A-Za-z0-9+/=]+$/.test(normalized)) {
return false;
}
return inferMimeFromBase64(normalized) !== null;
}
function toDataUrlFromBase64(base64: string, mime: string): string {
return `data:${mime};base64,${normalizeBase64(base64)}`;
}
function resolveImagePayloadObject(
value: unknown,
maxBytes: number,
): ImagePreviewResult | null {
if (!value || typeof value !== "object") {
return null;
}
const payload = value as PreviewImagePayload;
if (payload.type !== "image" || typeof payload.data !== "string") {
return null;
}
const mime = typeof payload.mime === "string" ? payload.mime : "image/jpeg";
const estimatedBytes = estimateBase64Bytes(payload.data);
if (estimatedBytes > maxBytes) {
return { kind: "too_large", estimatedBytes };
}
return {
kind: "ready",
src: toDataUrlFromBase64(payload.data, mime),
};
}
export function resolveImagePreview(
value: unknown,
maxBytes = MAX_IMAGE_PREVIEW_BYTES,
): ImagePreviewResult | null {
const payloadPreview = resolveImagePayloadObject(value, maxBytes);
if (payloadPreview) {
return payloadPreview;
}
if (typeof value !== "string") {
return null;
}
const trimmed = value.trim();
if (!trimmed) {
return null;
}
if (trimmed.startsWith("http://") || trimmed.startsWith("https://")) {
return { kind: "ready", src: trimmed };
}
if (trimmed.startsWith("data:image/")) {
const marker = "base64,";
const markerIdx = trimmed.indexOf(marker);
if (markerIdx < 0) {
return { kind: "ready", src: trimmed };
}
const encoded = trimmed.slice(markerIdx + marker.length);
const estimatedBytes = estimateBase64Bytes(encoded);
if (estimatedBytes > maxBytes) {
return { kind: "too_large", estimatedBytes };
}
return { kind: "ready", src: trimmed };
}
if (isLikelyRawBase64Image(trimmed)) {
const estimatedBytes = estimateBase64Bytes(trimmed);
if (estimatedBytes > maxBytes) {
return { kind: "too_large", estimatedBytes };
}
const mime = inferMimeFromBase64(trimmed) ?? "image/png";
return { kind: "ready", src: toDataUrlFromBase64(trimmed, mime) };
}
return null;
}
export function isLikelyImageValue(value: unknown): boolean {
return resolveImagePreview(value, Number.POSITIVE_INFINITY) !== null;
}

22
studio/frontend/src/features/recipe-studio/utils/import/parsers/llm-parser.ts
View file

@ -44,6 +44,26 @@ export function parseLlm(
})
: [];
let imageContext: LlmConfig["image_context"] = {
enabled: false,
// biome-ignore lint/style/useNamingConvention: api schema
column_name: "",
};
if (Array.isArray(column.multi_modal_context)) {
const first = column.multi_modal_context.find((entry) => isRecord(entry));
if (first && isRecord(first)) {
const modality = readString(first.modality);
const columnName = readString(first.column_name) ?? "";
if (modality === "image" && columnName) {
imageContext = {
enabled: true,
// biome-ignore lint/style/useNamingConvention: api schema
column_name: columnName,
};
}
}
}
return {
id,
kind: "llm",
@ -63,5 +83,7 @@ export function parseLlm(
// biome-ignore lint/style/useNamingConvention: api schema
tool_alias: readString(column.tool_alias) ?? "",
scores: llmType === "judge" ? scores : undefined,
// biome-ignore lint/style/useNamingConvention: ui schema
image_context: imageContext,
};
}

25
studio/frontend/src/features/recipe-studio/utils/payload/build-payload.ts
View file

@ -42,6 +42,7 @@ import {
validateTimedeltaConfigs,
validateUsedProviders,
} from "./validate";
import { isLikelyImageValue } from "../image-preview";
function pushUniqueJson(
label: string,
@ -110,6 +111,30 @@ export function buildRecipePayload(
continue;
}
if (config.kind === "llm") {
if (config.image_context?.enabled) {
const imageContext = config.image_context;
const columnName = imageContext.column_name.trim();
if (columnName) {
if (firstSeed?.seed_columns && firstSeed.seed_columns.length > 0) {
if (!firstSeed.seed_columns.includes(columnName)) {
errors.push(
`LLM ${config.name}: image context column '${columnName}' not found in seed columns.`,
);
}
}
const previewRows = firstSeed?.seed_preview_rows ?? [];
if (previewRows.length > 0) {
const hasImageLikeValue = previewRows.some((row) =>
isLikelyImageValue(row[columnName]),
);
if (!hasImageLikeValue) {
errors.push(
`LLM ${config.name}: image context column '${columnName}' has no image-like values in preview rows.`,
);
}
}
}
}
columns.push(buildLlmColumn(config, errors));
for (const provider of config.mcp_providers ?? []) {
const builtProvider = buildLlmMcpProvider(provider, errors);

24
studio/frontend/src/features/recipe-studio/utils/payload/builders-llm.ts
View file

@ -1,5 +1,27 @@
import type { LlmConfig, LlmMcpProviderConfig, LlmToolConfig } from "../../types";
function buildImageContext(
config: LlmConfig,
errors: string[],
): Array<Record<string, unknown>> | undefined {
const imageContext = config.image_context;
if (!imageContext?.enabled) {
return undefined;
}
const columnName = imageContext.column_name.trim();
if (!columnName) {
errors.push(`LLM ${config.name}: image context column is required.`);
return undefined;
}
return [
{
modality: "image",
// biome-ignore lint/style/useNamingConvention: api schema
column_name: columnName,
},
];
}
export function buildLlmColumn(
config: LlmConfig,
errors: string[],
@ -14,6 +36,8 @@ export function buildLlmColumn(
// biome-ignore lint/style/useNamingConvention: api schema
system_prompt: config.system_prompt || undefined,
// biome-ignore lint/style/useNamingConvention: api schema
multi_modal_context: buildImageContext(config, errors),
// biome-ignore lint/style/useNamingConvention: api schema
tool_alias: toolAlias || undefined,
};

5
studio/frontend/src/features/recipe-studio/utils/validation.ts
View file

@ -173,6 +173,11 @@ export function getConfigErrors(config: NodeConfig | null): string[] {
}
}
}
if (config.image_context?.enabled) {
if (!config.image_context.column_name.trim()) {
errors.push("Image context column is required.");
}
}
}
if (config.kind === "expression") {
if (!config.expr.trim()) {