ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

sheet icons and llm judge

Browse source
This commit is contained in:
shine1i 2026-02-05 11:38:11 +01:00
commit fbf5a30c77
11 changed files with 453 additions and 42 deletions
Download patch file Download diff file Expand all files Collapse all files

37
studio/frontend/AGENTS.md Normal file
View file

@ -0,0 +1,37 @@
# Repository Guidelines
## Project Structure & Module Organization
- `src/` is app code; entry is `src/main.tsx`, global styles in `src/index.css`.
- `src/app/` holds app shell and routing; `src/features/` is feature slices w/ public `index.ts` exports.
- Shared UI lives in `src/components/` (shadcn in `src/components/ui/`).
- Shared logic in `src/hooks/`, `src/stores/`, `src/utils/`, `src/lib/`, and types in `src/types/`.
- Static assets: `src/assets/` and `public/`.
- `test/` is a Python harness for payload validation and preview; not a JS test suite.
## Build, Test, and Development Commands
- `bun run dev`: start Vite dev server.
- `bun run build`: typecheck + build to `dist/`.
- `bun run preview`: serve the production build locally.
- `bun run lint`: ESLint checks for TS/React.
- `bun run typecheck`: `tsc` no-emit verification.
- `bun run biome:check` / `bun run biome:fix`: format + lint w/ Biome.
- Optional harness: `python test/scripts/validate_payload.py test/data/ui_payload.json`.
## Coding Style & Naming Conventions
- TypeScript + React, 2-space indent (Biome).
- Prefer explicit, compact code; avoid heavy abstraction.
- Use path alias `@/` for app imports.
- Feature boundaries enforced: import from `@/features/<name>` only, not deep paths.
- Components in `PascalCase`, hooks in `useCamelCase`, files in `kebab-case` or `camelCase` per local convention.
## Testing Guidelines
- No frontend test runner configured yet; add one if needed.
- `test/` is for API payload validation and preview flows; add samples as `test/data/ui_payload_*.json`.
## Commit & Pull Request Guidelines
- Commit history shows short, imperative messages; optional prefix like `refactor:`; keep it terse.
- PRs should include: clear summary, linked issue (if any), and UI screenshots/gifs for visual changes.
- Call out new deps, config, or required env changes in the PR body.
## Agent Notes
- Keep changes minimal, focused, and easy to review.

60
studio/frontend/src/features/canvas-lab/blocks/registry.tsx
View file

@ -1,8 +1,17 @@
import {
BalanceScaleIcon,
Clock01Icon,
CodeIcon,
Database02Icon,
Flowchart01Icon,
SparklesIcon,
CodeSimpleIcon,
DiceFaces03Icon,
EqualSignIcon,
FingerPrintIcon,
FunctionIcon,
Parabola02Icon,
PencilEdit02Icon,
Tag01Icon,
TagsIcon,
UserAccountIcon,
} from "@hugeicons/core-free-icons";
import type { ReactElement } from "react";
import type { LlmType, NodeConfig, SamplerConfig, SamplerType } from "../types";
@ -24,7 +33,7 @@ import { UuidDialog } from "../dialogs/samplers/uuid-dialog";
export type BlockKind = "sampler" | "llm" | "expression";
export type BlockType = SamplerType | LlmType | "expression";
type IconType = typeof Database02Icon;
type IconType = typeof CodeIcon;
type BlockGroup = {
kind: BlockKind;
@ -54,19 +63,19 @@ export const BLOCK_GROUPS: BlockGroup[] = [
kind: "sampler",
title: "Sampler",
description: "Numeric + categorical blocks.",
icon: Database02Icon,
icon: DiceFaces03Icon,
},
{
kind: "llm",
title: "LLM",
description: "Text + structured blocks.",
icon: SparklesIcon,
icon: PencilEdit02Icon,
},
{
kind: "expression",
title: "Expression",
description: "Derived columns with Jinja.",
icon: CodeIcon,
icon: FunctionIcon,
},
];
@ -76,7 +85,7 @@ const BLOCK_DEFINITIONS: BlockDefinition[] = [
type: "category",
title: "Category",
description: "Pick from a list of values.",
icon: Database02Icon,
icon: Tag01Icon,
createConfig: (id, existing) => makeSamplerConfig(id, "category", existing),
renderDialog: ({ config, onUpdate }) =>
config.kind === "sampler" && config.sampler_type === "category" ? (
@ -91,7 +100,7 @@ const BLOCK_DEFINITIONS: BlockDefinition[] = [
type: "subcategory",
title: "Subcategory",
description: "Map sub-values to a category.",
icon: Database02Icon,
icon: TagsIcon,
createConfig: (id, existing) =>
makeSamplerConfig(id, "subcategory", existing),
renderDialog: ({ config, categoryOptions, onUpdate }) =>
@ -108,7 +117,7 @@ const BLOCK_DEFINITIONS: BlockDefinition[] = [
type: "uniform",
title: "Uniform",
description: "Random number between low/high.",
icon: Database02Icon,
icon: EqualSignIcon,
createConfig: (id, existing) => makeSamplerConfig(id, "uniform", existing),
renderDialog: ({ config, onUpdate }) =>
config.kind === "sampler" && config.sampler_type === "uniform" ? (
@ -123,7 +132,7 @@ const BLOCK_DEFINITIONS: BlockDefinition[] = [
type: "gaussian",
title: "Gaussian",
description: "Normal distribution sampler.",
icon: Database02Icon,
icon: Parabola02Icon,
createConfig: (id, existing) => makeSamplerConfig(id, "gaussian", existing),
renderDialog: ({ config, onUpdate }) =>
config.kind === "sampler" && config.sampler_type === "gaussian" ? (
@ -138,7 +147,7 @@ const BLOCK_DEFINITIONS: BlockDefinition[] = [
type: "datetime",
title: "Datetime",
description: "Date/time range sampler.",
icon: Database02Icon,
icon: Clock01Icon,
createConfig: (id, existing) => makeSamplerConfig(id, "datetime", existing),
renderDialog: ({ config, onUpdate }) =>
config.kind === "sampler" && config.sampler_type === "datetime" ? (
@ -153,7 +162,7 @@ const BLOCK_DEFINITIONS: BlockDefinition[] = [
type: "uuid",
title: "UUID",
description: "UUID string sampler.",
icon: Database02Icon,
icon: FingerPrintIcon,
createConfig: (id, existing) => makeSamplerConfig(id, "uuid", existing),
renderDialog: ({ config, onUpdate }) =>
config.kind === "sampler" && config.sampler_type === "uuid" ? (
@ -168,7 +177,7 @@ const BLOCK_DEFINITIONS: BlockDefinition[] = [
type: "person",
title: "Person",
description: "Synthetic person sampler.",
icon: Database02Icon,
icon: UserAccountIcon,
createConfig: (id, existing) => makeSamplerConfig(id, "person", existing),
renderDialog: ({ config, onUpdate }) =>
config.kind === "sampler" &&
@ -185,7 +194,7 @@ const BLOCK_DEFINITIONS: BlockDefinition[] = [
type: "text",
title: "LLM Text",
description: "Free-form prompt generation.",
icon: SparklesIcon,
icon: PencilEdit02Icon,
createConfig: (id, existing) => makeLlmConfig(id, "text", existing),
renderDialog: ({ config, onUpdate }) =>
config.kind === "llm" && config.llm_type === "text" ? (
@ -200,7 +209,7 @@ const BLOCK_DEFINITIONS: BlockDefinition[] = [
type: "structured",
title: "LLM Structured",
description: "JSON output via schema.",
icon: Flowchart01Icon,
icon: CodeIcon,
createConfig: (id, existing) => makeLlmConfig(id, "structured", existing),
renderDialog: ({ config, onUpdate }) =>
config.kind === "llm" && config.llm_type === "structured" ? (
@ -215,7 +224,7 @@ const BLOCK_DEFINITIONS: BlockDefinition[] = [
type: "code",
title: "LLM Code",
description: "Generate code or SQL.",
icon: CodeIcon,
icon: CodeSimpleIcon,
createConfig: (id, existing) => makeLlmConfig(id, "code", existing),
renderDialog: ({ config, onUpdate }) =>
config.kind === "llm" && config.llm_type === "code" ? (
@ -225,12 +234,27 @@ const BLOCK_DEFINITIONS: BlockDefinition[] = [
/>
) : null,
},
{
kind: "llm",
type: "judge",
title: "LLM Judge",
description: "Score outputs with criteria.",
icon: BalanceScaleIcon,
createConfig: (id, existing) => makeLlmConfig(id, "judge", existing),
renderDialog: ({ config, onUpdate }) =>
config.kind === "llm" && config.llm_type === "judge" ? (
<LlmDialog
config={config}
onUpdate={(patch) => onUpdate(config.id, patch)}
/>
) : null,
},
{
kind: "expression",
type: "expression",
title: "Expression",
description: "Transform columns with Jinja.",
icon: CodeIcon,
icon: FunctionIcon,
createConfig: (id, existing) => makeExpressionConfig(id, existing),
renderDialog: ({ config, onUpdate }) =>
config.kind === "expression" ? (

8
studio/frontend/src/features/canvas-lab/canvas-lab-page.tsx
View file

@ -1,6 +1,7 @@
import {
Background,
BackgroundVariant,
Controls,
type EdgeTypes,
type Node,
type NodeTypes,
@ -52,10 +53,10 @@ function LayoutControls({
return (
<Panel position="top-left" className="m-3 flex items-center gap-2">
<Button size="sm" variant="secondary" onClick={handleLayout}>
<Button size="sm" className="corner-squircle" variant="secondary" onClick={handleLayout}>
Auto layout
</Button>
<Button size="sm" variant="outline" onClick={onToggleDirection}>
<Button size="sm" className="corner-squircle" variant="outline" onClick={onToggleDirection}>
{direction}
</Button>
</Panel>
@ -267,7 +268,7 @@ export function CanvasLabPage(): ReactElement {
</div>
</div>
<div
className="relative h-[75vh] w-full rounded-3xl border border-border/60 bg-white shadow-sm"
className="relative h-[75vh] w-full rounded-2xl corner-squircle border "
ref={setSheetContainer}
>
<ReactFlow
@ -308,6 +309,7 @@ export function CanvasLabPage(): ReactElement {
onAddExpression={addExpressionNode}
/>
</Panel>
<Controls position="bottom-left" />
</ReactFlow>
</div>
</main>

36
studio/frontend/src/features/canvas-lab/components/block-sheet.tsx
View file

@ -53,20 +53,26 @@ function BlockSheetButton({
title,
description,
onClick,
isActive = false,
}: {
icon: typeof Database02Icon;
title: string;
description: string;
onClick: () => void;
isActive?: boolean;
}): ReactElement {
return (
<button
type="button"
onClick={onClick}
className="flex w-full items-center gap-3 rounded-2xl border border-border/60 bg-white px-3 py-3 text-left transition hover:border-border hover:bg-muted/40"
className={`flex w-full items-center gap-3 bg-white px-3 py-3 text-left transition border-l-2 ${
isActive
? "border-emerald-500"
: "border-transparent hover:border-border/60"
}`}
>
<div className="flex size-9 items-center justify-center rounded-xl border border-border bg-muted/30 text-muted-foreground">
<HugeiconsIcon icon={icon} className="size-4" />
<div className="flex size-9 items-center justify-center rounded-xl text-foreground/70">
<HugeiconsIcon icon={icon} className="size-5" />
</div>
<div className="flex-1">
<p className="text-sm font-semibold text-foreground">{title}</p>
@ -90,9 +96,15 @@ export function BlockSheet({
}: BlockSheetProps): ReactElement {
const title = getSheetTitle(view);
return (
<Sheet>
<Sheet
onOpenChange={(open) => {
if (open) {
onViewChange("root");
}
}}
>
<SheetTrigger asChild={true}>
<Button size="icon-sm" variant="secondary">
<Button size="lg" className={"corner-squircle "} variant="secondary">
<HugeiconsIcon icon={PlusSignIcon} className="size-4" />
</Button>
</SheetTrigger>
@ -102,7 +114,7 @@ export function BlockSheet({
position="absolute"
overlayPosition="absolute"
className="absolute gap-0 p-0 shadow-none"
overlayClassName="bg-transparent pointer-events-none"
overlayClassName="bg-transparent pointer-events-none backdrop-blur-none supports-backdrop-filter:backdrop-blur-none"
>
<SheetHeader className="border-b border-border/60 px-6 py-5">
<div className="flex items-center gap-2">
@ -119,25 +131,28 @@ export function BlockSheet({
<SheetTitle>{title}</SheetTitle>
</div>
</SheetHeader>
<div className="px-6 py-4">
<div className=" py-4">
<div className="mt-4 flex flex-col gap-2">
{view === "root" &&
BLOCK_GROUPS.map((item) => (
BLOCK_GROUPS.map((item, index) => (
<BlockSheetButton
key={item.kind}
icon={item.icon}
title={item.title}
description={item.description}
isActive={index === 0}
onClick={() => onViewChange(item.kind)}
/>
))}
{view !== "root" &&
getBlocksForKind(VIEW_KIND[view] ?? "sampler").map((item) => (
getBlocksForKind(VIEW_KIND[view] ?? "sampler").map(
(item, index) => (
<BlockSheetButton
key={item.type}
icon={item.icon}
title={item.title}
description={item.description}
isActive={index === 0}
onClick={() => {
if (item.kind === "sampler") {
onAddSampler(item.type as SamplerType);
@ -148,7 +163,8 @@ export function BlockSheet({
}
}}
/>
))}
),
)}
</div>
</div>
</SheetContent>

62
studio/frontend/src/features/canvas-lab/components/canvas-node.tsx
View file

@ -1,36 +1,84 @@
import { cn } from "@/lib/utils";
import {
BalanceScaleIcon,
Clock01Icon,
CodeIcon,
Database02Icon,
SparklesIcon,
CodeSimpleIcon,
DiceFaces03Icon,
EqualSignIcon,
FingerPrintIcon,
FunctionIcon,
Parabola02Icon,
PencilEdit02Icon,
Tag01Icon,
TagsIcon,
UserAccountIcon,
} from "@hugeicons/core-free-icons";
import { HugeiconsIcon } from "@hugeicons/react";
import type { NodeProps } from "@xyflow/react";
import { Handle, Position, useUpdateNodeInternals } from "@xyflow/react";
import { type ReactElement, memo, useEffect } from "react";
import type { CanvasNode as CanvasNodeType } from "../types";
import type {
CanvasNode as CanvasNodeType,
LlmType,
SamplerType,
} from "../types";
type IconType = typeof CodeIcon;
const NODE_META = {
sampler: {
icon: Database02Icon,
tone: "bg-emerald-50 text-emerald-600 border-emerald-100",
},
llm: {
icon: SparklesIcon,
tone: "bg-purple-50 text-purple-600 border-purple-100",
},
expression: {
icon: CodeIcon,
tone: "bg-sky-50 text-sky-600 border-sky-100",
},
} as const;
const SAMPLER_ICONS: Record<SamplerType, IconType> = {
category: Tag01Icon,
subcategory: TagsIcon,
uniform: EqualSignIcon,
gaussian: Parabola02Icon,
datetime: Clock01Icon,
uuid: FingerPrintIcon,
person: UserAccountIcon,
person_from_faker: UserAccountIcon,
};
const LLM_ICONS: Record<LlmType, IconType> = {
text: PencilEdit02Icon,
structured: CodeIcon,
code: CodeSimpleIcon,
judge: BalanceScaleIcon,
};
function resolveNodeIcon(
kind: CanvasNodeType["data"]["kind"],
blockType: CanvasNodeType["data"]["blockType"],
): IconType {
if (kind === "sampler" && blockType in SAMPLER_ICONS) {
return SAMPLER_ICONS[blockType as SamplerType];
}
if (kind === "llm" && blockType in LLM_ICONS) {
return LLM_ICONS[blockType as LlmType];
}
if (kind === "expression") {
return FunctionIcon;
}
return DiceFaces03Icon;
}
function CanvasNodeBase({
id,
data,
selected,
}: NodeProps<CanvasNodeType>): ReactElement {
const meta = NODE_META[data.kind];
const icon = resolveNodeIcon(data.kind, data.blockType);
const layoutDirection = data.layoutDirection ?? "LR";
const isHorizontal = layoutDirection === "LR";
const updateNodeInternals = useUpdateNodeInternals();
@ -55,7 +103,7 @@ function CanvasNodeBase({
meta.tone,
)}
>
<HugeiconsIcon icon={meta.icon} className="size-4" />
<HugeiconsIcon icon={icon} className="size-4" />
</div>
<div>
<p className="text-sm font-semibold text-foreground">{data.title}</p>

162
studio/frontend/src/features/canvas-lab/dialogs/llm/llm-dialog.tsx
View file

@ -1,3 +1,4 @@
import { Button } from "@/components/ui/button";
import { Input } from "@/components/ui/input";
import {
Select,
@ -8,7 +9,7 @@ import {
} from "@/components/ui/select";
import { Textarea } from "@/components/ui/textarea";
import type { ReactElement } from "react";
import type { LlmConfig } from "../../types";
import type { LlmConfig, Score, ScoreOption } from "../../types";
import { NameField } from "../shared/name-field";
const CODE_LANG_OPTIONS = [
@ -41,12 +42,69 @@ export function LlmDialog({ config, onUpdate }: LlmDialogProps): ReactElement {
const promptId = `${config.id}-prompt`;
const outputFormatId = `${config.id}-output-format`;
const systemPromptId = `${config.id}-system-prompt`;
const scores = config.scores ?? [];
const updateField = <K extends keyof LlmConfig>(
key: K,
value: LlmConfig[K],
) => {
onUpdate({ [key]: value } as Partial<LlmConfig>);
};
const updateScores = (next: Score[]) => updateField("scores", next);
const updateScore = (index: number, patch: Partial<Score>) => {
updateScores(
scores.map((score, i) =>
i === index ? { ...score, ...patch } : score,
),
);
};
const removeScore = (index: number) => {
updateScores(scores.filter((_, i) => i !== index));
};
const addScore = () => {
updateScores([
...scores,
{
name: "",
description: "",
options: [
{ value: "1", description: "" },
{ value: "5", description: "" },
],
},
]);
};
const updateOption = (
scoreIndex: number,
optionIndex: number,
patch: Partial<ScoreOption>,
) => {
const score = scores[scoreIndex];
if (!score) {
return;
}
const nextOptions = score.options.map((option, i) =>
i === optionIndex ? { ...option, ...patch } : option,
);
updateScore(scoreIndex, { options: nextOptions });
};
const addOption = (scoreIndex: number) => {
const score = scores[scoreIndex];
if (!score) {
return;
}
updateScore(scoreIndex, {
options: [...score.options, { value: "", description: "" }],
});
};
const removeOption = (scoreIndex: number, optionIndex: number) => {
const score = scores[scoreIndex];
if (!score) {
return;
}
updateScore(scoreIndex, {
options: score.options.filter((_, i) => i !== optionIndex),
});
};
return (
<div className="space-y-4">
<NameField
@ -106,6 +164,108 @@ export function LlmDialog({ config, onUpdate }: LlmDialogProps): ReactElement {
onChange={(event) => updateField("prompt", event.target.value)}
/>
</div>
{config.llm_type === "judge" && (
<div className="space-y-3">
<div className="flex items-center justify-between">
<p className="text-xs font-semibold uppercase text-muted-foreground">
Scores
</p>
<Button type="button" size="xs" variant="outline" onClick={addScore}>
Add score
</Button>
</div>
{scores.length === 0 && (
<p className="text-xs text-muted-foreground">
Add at least one score to define evaluation criteria.
</p>
)}
{scores.map((score, index) => (
<div
key={`${config.id}-score-${index}`}
className="rounded-2xl border border-border/60 p-3"
>
<div className="flex items-start justify-between gap-2">
<div className="grid flex-1 gap-2">
<Input
className="nodrag"
placeholder="Score name (e.g., Relevance)"
value={score.name}
onChange={(event) =>
updateScore(index, { name: event.target.value })
}
/>
<Textarea
className="nodrag"
placeholder="Score description and scoring guide"
value={score.description}
onChange={(event) =>
updateScore(index, { description: event.target.value })
}
/>
</div>
<Button
type="button"
size="xs"
variant="ghost"
onClick={() => removeScore(index)}
>
Remove
</Button>
</div>
<div className="mt-3 space-y-2">
<div className="flex items-center justify-between">
<p className="text-xs font-semibold uppercase text-muted-foreground">
Options
</p>
<Button
type="button"
size="xs"
variant="outline"
onClick={() => addOption(index)}
>
Add option
</Button>
</div>
{score.options.map((option, optionIndex) => (
<div
key={`${config.id}-score-${index}-opt-${optionIndex}`}
className="flex items-start gap-2"
>
<Input
className="nodrag w-20"
placeholder="Value"
value={option.value}
onChange={(event) =>
updateOption(index, optionIndex, {
value: event.target.value,
})
}
/>
<Textarea
className="nodrag min-h-[2.5rem] flex-1"
placeholder="Description"
value={option.description}
onChange={(event) =>
updateOption(index, optionIndex, {
description: event.target.value,
})
}
/>
<Button
type="button"
size="xs"
variant="ghost"
onClick={() => removeOption(index, optionIndex)}
>
Remove
</Button>
</div>
))}
</div>
</div>
))}
</div>
)}
{config.llm_type === "structured" && (
<div className="grid gap-2">
<label

15
studio/frontend/src/features/canvas-lab/types/index.ts
View file

@ -10,7 +10,7 @@ export type SamplerType =
| "person"
| "person_from_faker";
export type LlmType = "text" | "structured" | "code";
export type LlmType = "text" | "structured" | "code" | "judge";
export type ExpressionDtype = "str" | "int" | "float" | "bool";
@ -21,6 +21,7 @@ export type CanvasNodeData = {
name: string;
kind: "sampler" | "llm" | "expression";
subtype: string;
blockType: SamplerType | LlmType | "expression";
layoutDirection?: LayoutDirection;
};
@ -64,6 +65,17 @@ export type SamplerConfig = {
subcategory_mapping?: Record<string, string[]>;
};
export type ScoreOption = {
value: string;
description: string;
};
export type Score = {
name: string;
description: string;
options: ScoreOption[];
};
export type LlmConfig = {
id: string;
kind: "llm";
@ -79,6 +91,7 @@ export type LlmConfig = {
code_lang?: string;
// biome-ignore lint/style/useNamingConvention: api schema
output_format?: string;
scores?: Score[];
};
export type ExpressionConfig = {

32
studio/frontend/src/features/canvas-lab/utils/import/parsers.ts
View file

@ -5,8 +5,15 @@ import type {
NodeConfig,
SamplerConfig,
SamplerType,
Score,
ScoreOption,
} from "../../types";
import { normalizeOutputFormat, readNumberString, readString } from "./helpers";
import {
isRecord,
normalizeOutputFormat,
readNumberString,
readString,
} from "./helpers";
const SAMPLER_TYPES: SamplerType[] = [
"category",
@ -186,7 +193,28 @@ function parseLlm(
llmType = "structured";
} else if (columnType === "llm-code") {
llmType = "code";
} else if (columnType === "llm-judge") {
llmType = "judge";
}
const scores: Score[] =
columnType === "llm-judge" && Array.isArray(column.scores)
? column.scores
.filter((score) => isRecord(score))
.map((score) => {
const options: ScoreOption[] = [];
const rawOptions = isRecord(score.options) ? score.options : {};
for (const [key, value] of Object.entries(rawOptions)) {
const description =
typeof value === "string" ? value : JSON.stringify(value);
options.push({ value: String(key), description });
}
return {
name: readString(score.name) ?? "",
description: readString(score.description) ?? "",
options,
};
})
: [];
return {
id,
kind: "llm",
@ -202,6 +230,7 @@ function parseLlm(
code_lang: readString(column.code_lang) ?? "",
// biome-ignore lint/style/useNamingConvention: api schema
output_format: normalizeOutputFormat(column.output_format),
scores: llmType === "judge" ? scores : undefined,
};
}
@ -236,6 +265,7 @@ const COLUMN_PARSERS: Record<string, ColumnParser> = {
"llm-text": (column, name, id) => parseLlm(column, name, id),
"llm-structured": (column, name, id) => parseLlm(column, name, id),
"llm-code": (column, name, id) => parseLlm(column, name, id),
"llm-judge": (column, name, id) => parseLlm(column, name, id),
};
export function parseColumn(

25
studio/frontend/src/features/canvas-lab/utils/index.ts
View file

@ -26,6 +26,7 @@ const LLM_LABELS: Record<LlmType, string> = {
text: "LLM Text",
structured: "LLM Structured",
code: "LLM Code",
judge: "LLM Judge",
};
const EXPRESSION_LABELS: Record<ExpressionDtype, string> = {
@ -179,6 +180,8 @@ export function makeLlmConfig(
namePrefix = "llm_structured";
} else if (llmType === "code") {
namePrefix = "llm_code";
} else if (llmType === "judge") {
namePrefix = "llm_judge";
}
const name = nextName(existing, namePrefix);
return {
@ -189,7 +192,10 @@ export function makeLlmConfig(
name,
// biome-ignore lint/style/useNamingConvention: api schema
model_alias: "allenai/olmo-3.1-32b-instruct",
prompt: "Write a response.",
prompt:
llmType === "judge"
? "Evaluate the content using the scoring criteria below."
: "Write a response.",
// biome-ignore lint/style/useNamingConvention: api schema
system_prompt: "",
// biome-ignore lint/style/useNamingConvention: api schema
@ -197,6 +203,20 @@ export function makeLlmConfig(
// biome-ignore lint/style/useNamingConvention: api schema
output_format:
llmType === "structured" ? '{\n "field": "string"\n}' : undefined,
scores:
llmType === "judge"
? [
{
name: "Quality",
description: "Overall quality based on the criteria.",
options: [
{ value: "1", description: "Poor" },
{ value: "3", description: "Acceptable" },
{ value: "5", description: "Excellent" },
],
},
]
: undefined,
};
}
@ -234,6 +254,7 @@ export function nodeDataFromConfig(
title: "Sampler",
kind: "sampler",
subtype: labelForSampler(config.sampler_type),
blockType: config.sampler_type,
name: config.name,
layoutDirection,
};
@ -243,6 +264,7 @@ export function nodeDataFromConfig(
title: "Expression",
kind: "expression",
subtype: labelForExpression(config.dtype),
blockType: "expression",
name: config.name,
layoutDirection,
};
@ -251,6 +273,7 @@ export function nodeDataFromConfig(
title: "LLM",
kind: "llm",
subtype: labelForLlm(config.llm_type),
blockType: config.llm_type,
name: config.name,
layoutDirection,
};

32
studio/frontend/src/features/canvas-lab/utils/payload.ts
View file

@ -216,6 +216,38 @@ function buildLlmColumn(
output_format: outputFormat,
};
}
if (config.llm_type === "judge") {
const scores = (config.scores ?? [])
.map((score) => {
const options: Record<string, string> = {};
for (const option of score.options ?? []) {
const key = option.value.trim();
const value = option.description.trim();
if (!key || !value) {
continue;
}
options[key] = value;
}
return {
name: score.name.trim(),
description: score.description.trim(),
options,
};
})
.filter(
(score) =>
score.name && score.description && Object.keys(score.options).length > 0,
);
if (scores.length === 0) {
errors.push(`LLM ${config.name}: scores required for LLM Judge.`);
}
return {
// biome-ignore lint/style/useNamingConvention: api schema
column_type: "llm-judge",
...base,
scores,
};
}
return {
// biome-ignore lint/style/useNamingConvention: api schema
column_type: "llm-text",

26
studio/frontend/src/features/canvas-lab/utils/validation.ts
View file

@ -124,6 +124,32 @@ export function getConfigErrors(config: NodeConfig | null): string[] {
}
}
}
if (config.llm_type === "judge") {
const scores = config.scores ?? [];
if (scores.length === 0) {
errors.push("LLM Judge needs at least one score.");
}
for (const score of scores) {
if (!score.name.trim()) {
errors.push("LLM Judge score name is required.");
}
if (!score.description.trim()) {
errors.push("LLM Judge score description is required.");
}
const options = score.options ?? [];
if (options.length === 0) {
errors.push(`LLM Judge score ${score.name || "Unnamed"} needs options.`);
}
for (const option of options) {
if (!option.value.trim() || !option.description.trim()) {
errors.push(
`LLM Judge score ${score.name || "Unnamed"} options need value + description.`,
);
break;
}
}
}
}
}
if (config.kind === "expression") {
if (!config.expr.trim()) {