ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

My changes for dataset

Browse source
This commit is contained in:
Manan17 2026-02-25 08:15:44 +00:00
commit 47fc79df6d
8 changed files with 173 additions and 163 deletions
Download patch file Download diff file Expand all files Collapse all files

2
studio/backend/utils/datasets/dataset_utils.py
View file

@ -507,7 +507,7 @@ def format_dataset(
}
# CHATML MODE: Convert to ChatML
elif format_type in ["chatml", "conversational"]:
elif format_type in ["chatml", "conversational", "sharegpt"]:
if detected["format"] == "alpaca":
converted = convert_alpaca_to_chatml(dataset, batch_size, num_proc)

6
studio/frontend/src/features/onboarding/components/steps/dataset-step.tsx
View file

@ -75,6 +75,8 @@ export function DatasetStep() {
setDatasetSubset,
datasetSplit,
setDatasetSplit,
datasetEvalSplit,
setDatasetEvalSplit,
uploadedFile,
setUploadedFile,
} = useTrainingConfigStore(
@ -91,6 +93,8 @@ export function DatasetStep() {
setDatasetSubset: s.setDatasetSubset,
datasetSplit: s.datasetSplit,
setDatasetSplit: s.setDatasetSplit,
datasetEvalSplit: s.datasetEvalSplit,
setDatasetEvalSplit: s.setDatasetEvalSplit,
uploadedFile: s.uploadedFile,
setUploadedFile: s.setUploadedFile,
})),
@ -304,6 +308,8 @@ export function DatasetStep() {
setDatasetSubset={setDatasetSubset}
datasetSplit={datasetSplit}
setDatasetSplit={setDatasetSplit}
datasetEvalSplit={datasetEvalSplit}
setDatasetEvalSplit={setDatasetEvalSplit}
/>
</>
) : (

6
studio/frontend/src/features/studio/sections/dataset-section.tsx
View file

@ -66,6 +66,8 @@ export function DatasetSection() {
setDatasetSubset,
datasetSplit,
setDatasetSplit,
datasetEvalSplit,
setDatasetEvalSplit,
hfToken,
} = useTrainingConfigStore(
useShallow((s) => ({
@ -77,6 +79,8 @@ export function DatasetSection() {
setDatasetSubset: s.setDatasetSubset,
datasetSplit: s.datasetSplit,
setDatasetSplit: s.setDatasetSplit,
datasetEvalSplit: s.datasetEvalSplit,
setDatasetEvalSplit: s.setDatasetEvalSplit,
hfToken: s.hfToken,
})),
);
@ -282,6 +286,8 @@ export function DatasetSection() {
setDatasetSubset={setDatasetSubset}
datasetSplit={datasetSplit}
setDatasetSplit={setDatasetSplit}
datasetEvalSplit={datasetEvalSplit}
setDatasetEvalSplit={setDatasetEvalSplit}
/>
<div className="flex flex-col gap-2">

5
studio/frontend/src/features/training/api/mappers.ts
View file

@ -24,8 +24,9 @@ export function buildTrainingStartPayload(
load_in_4bit: adapterMethod ? isQlorMethod : false,
max_seq_length: config.contextLength,
hf_dataset: hfDataset,
hf_dataset_config: hfDataset ? config.datasetSubset : null,
hf_dataset_split: hfDataset ? config.datasetSplit : null,
subset: hfDataset ? config.datasetSubset : null,
train_split: hfDataset ? config.datasetSplit : null,
eval_split: hfDataset ? config.datasetEvalSplit : null,
local_datasets: [],
format_type: config.datasetFormat,
custom_format_mapping: customFormatMapping,

300
studio/frontend/src/features/training/components/hf-dataset-subset-split-selectors.tsx
View file

@ -29,6 +29,8 @@ type Props = {
setDatasetSubset: (v: string | null) => void;
datasetSplit: string | null;
setDatasetSplit: (v: string | null) => void;
datasetEvalSplit: string | null;
setDatasetEvalSplit: (v: string | null) => void;
};
export function HfDatasetSubsetSplitSelectors({
@ -40,12 +42,13 @@ export function HfDatasetSubsetSplitSelectors({
setDatasetSubset,
datasetSplit,
setDatasetSplit,
datasetEvalSplit,
setDatasetEvalSplit,
}: Props) {
const {
subsets: hfSubsets,
splits: hfSplits,
hasMultipleSubsets,
hasMultipleSplits,
isLoading,
error,
} = useHfDatasetSplits(enabled ? datasetName : null, datasetSubset, {
@ -78,6 +81,8 @@ export function HfDatasetSubsetSplitSelectors({
if (!enabled || !datasetName) return null;
const showDropdowns = !isLoading && !error && hfSubsets.length > 0;
return (
<>
{isLoading && (
@ -105,167 +110,144 @@ export function HfDatasetSubsetSplitSelectors({
</div>
)}
{!isLoading && !error && hasMultipleSubsets && (
{showDropdowns && (
<>
{variant === "wizard" ? (
<Field>
<FieldLabel className="flex items-center gap-1.5">
Subset
<Tooltip>
<TooltipTrigger asChild={true}>
<button
type="button"
className="text-muted-foreground/50 hover:text-muted-foreground"
>
<HugeiconsIcon
icon={InformationCircleIcon}
className="size-3.5"
/>
</button>
</TooltipTrigger>
<TooltipContent className="max-w-xs">
This dataset has multiple subsets. Select which one to use
for training.
</TooltipContent>
</Tooltip>
</FieldLabel>
<Select
value={datasetSubset ?? ""}
onValueChange={(v) => setDatasetSubset(v || null)}
>
<SelectTrigger className="w-full">
<SelectValue placeholder="Select a subset..." />
</SelectTrigger>
<SelectContent>
{hfSubsets.map((subset) => (
<SelectItem key={subset} value={subset}>
{subset}
</SelectItem>
))}
</SelectContent>
</Select>
</Field>
) : (
<div className="flex flex-col gap-1.5">
<span className="flex items-center gap-1.5 text-xs font-medium text-muted-foreground">
Subset
<Tooltip>
<TooltipTrigger asChild={true}>
<button
type="button"
className="text-foreground/70 hover:text-foreground"
>
<HugeiconsIcon
icon={InformationCircleIcon}
className="size-3"
/>
</button>
</TooltipTrigger>
<TooltipContent>
This dataset has multiple subsets. Select which one to use
for training.
</TooltipContent>
</Tooltip>
</span>
<Select
value={datasetSubset ?? ""}
onValueChange={(v) => setDatasetSubset(v || null)}
>
<SelectTrigger className="w-full">
<SelectValue placeholder="Select a subset..." />
</SelectTrigger>
<SelectContent>
{hfSubsets.map((subset) => (
<SelectItem key={subset} value={subset}>
{subset}
</SelectItem>
))}
</SelectContent>
</Select>
</div>
)}
</>
)}
{!isLoading && !error && hasMultipleSplits && (
<>
{variant === "wizard" ? (
<Field>
<FieldLabel className="flex items-center gap-1.5">
Split
<Tooltip>
<TooltipTrigger asChild={true}>
<button
type="button"
className="text-muted-foreground/50 hover:text-muted-foreground"
>
<HugeiconsIcon
icon={InformationCircleIcon}
className="size-3.5"
/>
</button>
</TooltipTrigger>
<TooltipContent className="max-w-xs">
Select which split of the dataset to use for training.
</TooltipContent>
</Tooltip>
</FieldLabel>
<Select
value={datasetSplit ?? ""}
onValueChange={(v) => setDatasetSplit(v || null)}
>
<SelectTrigger className="w-full">
<SelectValue placeholder="Select a split..." />
</SelectTrigger>
<SelectContent>
{hfSplits.map((split) => (
<SelectItem key={split} value={split}>
{split}
</SelectItem>
))}
</SelectContent>
</Select>
</Field>
) : (
<div className="flex flex-col gap-1.5">
<span className="flex items-center gap-1.5 text-xs font-medium text-muted-foreground">
Split
<Tooltip>
<TooltipTrigger asChild={true}>
<button
type="button"
className="text-foreground/70 hover:text-foreground"
>
<HugeiconsIcon
icon={InformationCircleIcon}
className="size-3"
/>
</button>
</TooltipTrigger>
<TooltipContent>
Select which split of the dataset to use for training.
</TooltipContent>
</Tooltip>
</span>
<Select
value={datasetSplit ?? ""}
onValueChange={(v) => setDatasetSplit(v || null)}
>
<SelectTrigger className="w-full">
<SelectValue placeholder="Select a split..." />
</SelectTrigger>
<SelectContent>
{hfSplits.map((split) => (
<SelectItem key={split} value={split}>
{split}
</SelectItem>
))}
</SelectContent>
</Select>
</div>
)}
<SelectorDropdown
variant={variant}
label="Subset"
tooltip="Select which subset (config) of the dataset to use."
value={datasetSubset}
onChange={setDatasetSubset}
options={hfSubsets}
placeholder="Select a subset..."
/>
<SelectorDropdown
variant={variant}
label="Train Split"
tooltip="Select which split to use for training."
value={datasetSplit}
onChange={setDatasetSplit}
options={hfSplits}
placeholder="Select a split..."
/>
<SelectorDropdown
variant={variant}
label="Eval Split"
tooltip="Select which split to use for evaluation. None means no evaluation during training."
value={datasetEvalSplit}
onChange={setDatasetEvalSplit}
options={hfSplits}
placeholder="None"
allowNone
/>
</>
)}
</>
);
}
function SelectorDropdown({
variant,
label,
tooltip,
value,
onChange,
options,
placeholder,
allowNone = false,
}: {
variant: "wizard" | "studio";
label: string;
tooltip: string;
value: string | null;
onChange: (v: string | null) => void;
options: string[];
placeholder: string;
allowNone?: boolean;
}) {
if (variant === "wizard") {
return (
<Field>
<FieldLabel className="flex items-center gap-1.5">
{label}
<Tooltip>
<TooltipTrigger asChild={true}>
<button
type="button"
className="text-muted-foreground/50 hover:text-muted-foreground"
>
<HugeiconsIcon
icon={InformationCircleIcon}
className="size-3.5"
/>
</button>
</TooltipTrigger>
<TooltipContent className="max-w-xs">
{tooltip}
</TooltipContent>
</Tooltip>
</FieldLabel>
<Select
value={value ?? "_none"}
onValueChange={(v) => onChange(v === "_none" ? null : v)}
>
<SelectTrigger className="w-full">
<SelectValue placeholder={placeholder} />
</SelectTrigger>
<SelectContent>
{allowNone && (
<SelectItem value="_none">None</SelectItem>
)}
{options.map((opt) => (
<SelectItem key={opt} value={opt}>
{opt}
</SelectItem>
))}
</SelectContent>
</Select>
</Field>
);
}
return (
<div className="flex flex-col gap-1.5">
<span className="flex items-center gap-1.5 text-xs font-medium text-muted-foreground">
{label}
<Tooltip>
<TooltipTrigger asChild={true}>
<button
type="button"
className="text-foreground/70 hover:text-foreground"
>
<HugeiconsIcon
icon={InformationCircleIcon}
className="size-3"
/>
</button>
</TooltipTrigger>
<TooltipContent>
{tooltip}
</TooltipContent>
</Tooltip>
</span>
<Select
value={value ?? "_none"}
onValueChange={(v) => onChange(v === "_none" ? null : v)}
>
<SelectTrigger className="w-full">
<SelectValue placeholder={placeholder} />
</SelectTrigger>
<SelectContent>
{allowNone && (
<SelectItem value="_none">None</SelectItem>
)}
{options.map((opt) => (
<SelectItem key={opt} value={opt}>
{opt}
</SelectItem>
))}
</SelectContent>
</Select>
</div>
);
}

14
studio/frontend/src/features/training/stores/training-config-store.ts
View file

@ -26,6 +26,7 @@ const initialState: TrainingConfigState = {
dataset: null,
datasetSubset: null,
datasetSplit: null,
datasetEvalSplit: null,
datasetManualMapping: emptyManualMapping(),
uploadedFile: null,
isCheckingVision: false,
@ -252,6 +253,7 @@ export const useTrainingConfigStore = create<TrainingConfigStore>()(
dataset,
datasetSubset: null,
datasetSplit: null,
datasetEvalSplit: null,
datasetManualMapping: emptyManualMapping(),
isDatasetMultimodal: null,
isCheckingDataset: false,
@ -264,6 +266,7 @@ export const useTrainingConfigStore = create<TrainingConfigStore>()(
set({
datasetSubset,
datasetSplit: null,
datasetEvalSplit: null,
datasetManualMapping: emptyManualMapping(),
isDatasetMultimodal: null,
isCheckingDataset: false,
@ -300,6 +303,12 @@ export const useTrainingConfigStore = create<TrainingConfigStore>()(
const split = state.datasetSplit || "train";
runDatasetCheck(datasetName, split);
},
setDatasetEvalSplit: (datasetEvalSplit) => {
set({
datasetEvalSplit,
evalSteps: datasetEvalSplit ? 0.1 : 0,
});
},
setDatasetManualMapping: (datasetManualMapping) =>
set({ datasetManualMapping }),
setUploadedFile: (uploadedFile) => set({ uploadedFile }),
@ -359,7 +368,7 @@ export const useTrainingConfigStore = create<TrainingConfigStore>()(
},
{
name: "unsloth_training_config_v1",
version: 5,
version: 6,
migrate: (persisted, version) => {
const s = persisted as Record<string, unknown>;
if (version < 2 && s.datasetSubset == null && s.datasetConfig != null) {
@ -375,6 +384,9 @@ export const useTrainingConfigStore = create<TrainingConfigStore>()(
if (version < 5 && s.lrSchedulerType == null) {
s.lrSchedulerType = DEFAULT_HYPERPARAMS.lrSchedulerType;
}
if (version < 6 && s.datasetEvalSplit == null) {
s.datasetEvalSplit = null;
}
return s as unknown as TrainingConfigStore;
},
partialize: partializePersistedState,

5
studio/frontend/src/features/training/types/api.ts
View file

@ -5,8 +5,9 @@ export interface TrainingStartRequest {
load_in_4bit: boolean;
max_seq_length: number;
hf_dataset: string | null;
hf_dataset_config: string | null;
hf_dataset_split: string | null;
subset: string | null;
train_split: string | null;
eval_split: string | null;
local_datasets: string[];
format_type: string;
custom_format_mapping?: Record<string, string> | null;

2
studio/frontend/src/features/training/types/config.ts
View file

@ -24,6 +24,7 @@ export interface TrainingConfigState {
dataset: string | null;
datasetSubset: string | null;
datasetSplit: string | null;
datasetEvalSplit: string | null;
datasetManualMapping: DatasetManualMapping;
uploadedFile: string | null;
epochs: number;
@ -81,6 +82,7 @@ export interface TrainingConfigActions {
setDataset: (dataset: string | null) => void;
setDatasetSubset: (subset: string | null) => void;
setDatasetSplit: (split: string | null) => void;
setDatasetEvalSplit: (split: string | null) => void;
setDatasetManualMapping: (mapping: DatasetManualMapping) => void;
setUploadedFile: (file: string | null) => void;
setEpochs: (epochs: number) => void;