merging with nightly

studio/backend/assets/configs/model_defaults/other/OuteAI_Llama-OuteTTS-1.0-1B.yaml

@@ -3,7 +3,10 @@
 # Also applies to: OuteAI/Llama-OuteTTS-1.0-1B
 # added inference parameters from unsloth notebook
 
+audio_type: dac
+
 training:
+  eval_steps: 0
   max_seq_length: 2048
   # num_epochs: 4
   num_epochs: 0

studio/backend/assets/configs/model_defaults/other/Spark-TTS-0.5B_LLM.yaml

@@ -3,7 +3,10 @@
 # Also applies to: Spark-TTS-0.5B/LLM
 # added inference parameters from unsloth notebook
 
+audio_type: bicodec
+
 training:
+  eval_steps: 0
   max_seq_length: 2048
   # num_epochs: 4
   num_epochs: 0

studio/backend/assets/configs/model_defaults/other/sesame_csm-1b.yaml

@@ -2,7 +2,10 @@
 # Based on Sesame_CSM_(1B)-TTS.ipynb
 # Also applies to: sesame/csm-1b
 
+audio_type: csm
+
 training:
+  eval_steps: 0
   max_seq_length: 2048
   # num_epochs: 4
   num_epochs: 0

studio/backend/assets/configs/model_defaults/other/unsloth_orpheus-3b-0.1-ft.yaml

@@ -3,7 +3,10 @@
 # Also applies to: unsloth/orpheus-3b-0.1-ft-unsloth-bnb-4bit, canopylabs/orpheus-3b-0.1-ft, unsloth/orpheus-3b-0.1-ft-bnb-4bit
 # added inference parameters from unsloth notebook
 
+audio_type: snac
+
 training:
+  eval_steps: 0
   max_seq_length: 2048
   # num_epochs: 4
   num_epochs: 0

studio/backend/assets/configs/model_defaults/other/unsloth_whisper-large-v3.yaml

@@ -2,7 +2,10 @@
 # Based on Whisper.ipynb
 # Also applies to: unsloth/whisper-large-v3, openai/whisper-large-v3
 
+audio_type: whisper
+
 training:
+  eval_steps: 5
   max_seq_length: 448
   # num_epochs: 4
   num_epochs: 0

studio/backend/core/training/training.py

@@ -116,7 +116,8 @@ class TrainingBackend:
                        train_split: str = "train",
                        eval_split: str = None,
                        eval_steps: float = 0.00,
-                       is_dataset_multimodal: bool = False) -> bool:
+                       is_dataset_multimodal: bool = False,
+                       is_dataset_audio: bool = False) -> bool:
         """
         Start training.
 
@@ -145,6 +146,11 @@ class TrainingBackend:
             import torch as _torch
             if _torch.cuda.is_available():
                 _torch.cuda.synchronize()
+            # Reset torch dynamo/compiler caches — Unsloth's compiled SFTTrainer
+            # and model.for_training() set class-level state that persists between
+            # runs (e.g. BiCodec text trainer pollutes subsequent VLM runs).
+            _torch._dynamo.reset()
+            _torch.compiler.reset()
             import gc
             gc.collect()
             clear_gpu_cache()
@@ -177,6 +183,7 @@ class TrainingBackend:
                 load_in_4bit=load_in_4bit if use_lora_actual else False,  # Only 4bit for LoRA
                 hf_token=hf_token if hf_token.strip() else None,
                 is_dataset_multimodal=is_dataset_multimodal,
+                is_dataset_audio=is_dataset_audio,
             )
 
             if not success or self.trainer.should_stop:

studio/backend/models/training.py

@@ -66,6 +66,7 @@ class TrainingStartRequest(BaseModel):
     finetune_attention_modules: bool = Field(False, description="Finetune attention modules")
     finetune_mlp_modules: bool = Field(False, description="Finetune MLP modules")
     is_dataset_multimodal: bool = Field(False, description="Whether the dataset contains multimodal (image) data")
+    is_dataset_audio: bool = Field(False, description="Whether the dataset contains audio data")
 
     # Logging parameters
     enable_wandb: bool = Field(False, description="Enable Weights & Biases logging")

studio/backend/routes/training.py

@@ -179,6 +179,7 @@ async def start_training(
             "finetune_attention_modules": request.finetune_attention_modules,
             "finetune_mlp_modules": request.finetune_mlp_modules,
             "is_dataset_multimodal": request.is_dataset_multimodal,
+            "is_dataset_audio": request.is_dataset_audio,
             "enable_wandb": request.enable_wandb,
             "wandb_token": request.wandb_token or "",
             "wandb_project": request.wandb_project or "",

studio/backend/utils/datasets/__init__.py

@@ -45,6 +45,7 @@ from .vlm_processing import (
 
 # Data collators
 from .data_collators import (
+    DataCollatorSpeechSeq2SeqWithPadding,
     DeepSeekOCRDataCollator,
     VLMDataCollator,
 )
@@ -85,6 +86,7 @@ __all__ = [
     # VLM
     "generate_smart_vlm_instruction",
     # Collators
+    "DataCollatorSpeechSeq2SeqWithPadding",
     "DeepSeekOCRDataCollator",
     "VLMDataCollator",
     # Mappings

studio/backend/utils/datasets/data_collators.py

@@ -10,6 +10,33 @@ from dataclasses import dataclass
 from typing import Any, List, Optional, Union
 
 
+@dataclass
+class DataCollatorSpeechSeq2SeqWithPadding:
+    """
+    Data collator for Whisper speech-to-text training.
+
+    Pads input features (audio) and label sequences (text) separately,
+    masks padding in labels with -100, and strips leading BOS token.
+    Mirrors the collator from the Whisper.ipynb notebook.
+    """
+    processor: Any
+
+    def __call__(self, features: List[dict]) -> dict:
+        input_features = [{"input_features": feature["input_features"]} for feature in features]
+        batch = self.processor.feature_extractor.pad(input_features, return_tensors="pt")
+
+        label_features = [{"input_ids": feature["labels"]} for feature in features]
+        labels_batch = self.processor.tokenizer.pad(label_features, return_tensors="pt")
+
+        labels = labels_batch["input_ids"].masked_fill(labels_batch.attention_mask.ne(1), -100)
+
+        if (labels[:, 0] == self.processor.tokenizer.bos_token_id).all().cpu().item():
+            labels = labels[:, 1:]
+
+        batch["labels"] = labels
+        return batch
+
+
 @dataclass
 class DeepSeekOCRDataCollator:
     """

studio/backend/utils/models/model_config.py

@@ -213,6 +213,7 @@ MODEL_NAME_MAPPING = {
         "unsloth/Nemotron-3-Nano-30B-A3B",
     ],
     "unsloth_orpheus-3b-0.1-ft.yaml": [
+        "unsloth/orpheus-3b-0.1-ft",
         "unsloth/orpheus-3b-0.1-ft-unsloth-bnb-4bit",
         "canopylabs/orpheus-3b-0.1-ft",
         "unsloth/orpheus-3b-0.1-ft-bnb-4bit",
@@ -317,9 +318,11 @@ MODEL_NAME_MAPPING = {
     ],
     "sesame_csm-1b.yaml": [
         "sesame/csm-1b",
+        "unsloth/csm-1b",
     ],
     "Spark-TTS-0.5B_LLM.yaml": [
         "Spark-TTS-0.5B/LLM",
+        "unsloth/Spark-TTS-0.5B",
     ],
     "unsloth_tinyllama-bnb-4bit.yaml": [
         "unsloth/tinyllama",

studio/frontend/src/features/studio/sections/training-section.tsx

@@ -42,8 +42,8 @@ export function TrainingSection() {
   const store = useTrainingConfigStore();
   const { isStarting, startError, startTrainingRun } = useTrainingActions();
   const isIncompatible =
-    !store.isVisionModel && store.isDatasetMultimodal === true;
-  const fileInputRef = useRef<HTMLInputElement>(null);
+    !store.isVisionModel && !store.isDatasetAudio && store.isDatasetMultimodal === true;
+    const fileInputRef = useRef<HTMLInputElement>(null);
 
   const handleFileUpload = (e: React.ChangeEvent<HTMLInputElement>) => {
     const file = e.target.files?.[0];

studio/frontend/src/features/training/api/mappers.ts

@@ -58,6 +58,7 @@ export function buildTrainingStartPayload(
     finetune_attention_modules: config.finetuneAttentionModules,
     finetune_mlp_modules: config.finetuneMLPModules,
     is_dataset_multimodal: !!config.isDatasetMultimodal,
+    is_dataset_audio: config.isDatasetAudio,
     enable_wandb: config.enableWandb,
     wandb_token: config.enableWandb ? config.wandbToken.trim() || null : null,
     wandb_project: config.enableWandb

studio/frontend/src/features/training/api/models-api.ts

@@ -46,6 +46,7 @@ interface BackendLoggingDefaults {
 }
 
 export interface BackendModelConfig {
+  audio_type?: string | null;
   training?: BackendTrainingDefaults;
   lora?: BackendLoraDefaults;
   logging?: BackendLoggingDefaults;

studio/frontend/src/features/training/lib/model-defaults.ts

@@ -4,6 +4,7 @@ import type { TrainingConfigState } from "../types/config";
 type ModelDefaultsPatch = Partial<
   Pick<
     TrainingConfigState,
+    | "isDatasetAudio"
     | "epochs"
     | "contextLength"
     | "learningRate"
@@ -79,6 +80,9 @@ export function mapBackendModelConfigToTrainingPatch(
   const lora = config.lora;
   const logging = config.logging;
 
+  // Audio models: set isDatasetAudio based on audio_type from YAML
+  patch.isDatasetAudio = typeof config.audio_type === "string" && config.audio_type.length > 0;
+
   const maxSeqLength = toNumber(training?.max_seq_length);
   if (maxSeqLength !== undefined) patch.contextLength = maxSeqLength;
 

studio/frontend/src/features/training/stores/training-config-store.ts

@@ -36,6 +36,7 @@ const initialState: TrainingConfigState = {
   modelDefaultsAppliedFor: null,
   isCheckingDataset: false,
   isDatasetMultimodal: null,
+  isDatasetAudio: false,
   ...DEFAULT_HYPERPARAMS,
 };
 
@@ -57,6 +58,7 @@ const NON_PERSISTED_STATE_KEYS: ReadonlySet<keyof TrainingConfigState> = new Set
   "modelDefaultsAppliedFor",
   "isCheckingDataset",
   "isDatasetMultimodal",
+  "isDatasetAudio",
   "trainOnCompletions",
 ]);
 
@@ -205,6 +207,7 @@ export const useTrainingConfigStore = create<TrainingConfigStore>()(
             selectedModel: null,
             isCheckingVision: false,
             isVisionModel: false,
+            isDatasetAudio: false,
             isLoadingModelDefaults: false,
             modelDefaultsError: null,
             modelDefaultsAppliedFor: null,
@@ -220,6 +223,7 @@ export const useTrainingConfigStore = create<TrainingConfigStore>()(
             set({
               isCheckingVision: false,
               isVisionModel: false,
+              isDatasetAudio: false,
               isLoadingModelDefaults: false,
               modelDefaultsError: null,
               modelDefaultsAppliedFor: null,

studio/frontend/src/features/training/types/api.ts

@@ -39,6 +39,7 @@ export interface TrainingStartRequest {
   finetune_attention_modules: boolean;
   finetune_mlp_modules: boolean;
   is_dataset_multimodal: boolean;
+  is_dataset_audio: boolean;
   enable_wandb: boolean;
   wandb_token: string | null;
   wandb_project: string | null;

studio/frontend/src/features/training/types/config.ts

@@ -60,6 +60,7 @@ export interface TrainingConfigState {
   modelDefaultsAppliedFor: string | null;
   isCheckingDataset: boolean;
   isDatasetMultimodal: boolean | null;
+  isDatasetAudio: boolean;
   finetuneVisionLayers: boolean;
   finetuneLanguageLayers: boolean;
   finetuneAttentionModules: boolean;