diff --git a/studio/backend/assets/configs/model_defaults/default.yaml b/studio/backend/assets/configs/model_defaults/default.yaml index 523775cff7..3a2563a6f9 100644 --- a/studio/backend/assets/configs/model_defaults/default.yaml +++ b/studio/backend/assets/configs/model_defaults/default.yaml @@ -45,3 +45,9 @@ logging: tensorboard_dir: "runs" log_frequency: 10 +inference: + temperature: 0.7 + top_p: 0.95 + top_k: -1 + min_p: 0.01 + diff --git a/studio/backend/assets/configs/model_defaults/ernie/unsloth_ERNIE-4.5-VL-28B-A3B-PT.yaml b/studio/backend/assets/configs/model_defaults/ernie/unsloth_ERNIE-4.5-VL-28B-A3B-PT.yaml index 04ae59d293..d2aeb9492e 100644 --- a/studio/backend/assets/configs/model_defaults/ernie/unsloth_ERNIE-4.5-VL-28B-A3B-PT.yaml +++ b/studio/backend/assets/configs/model_defaults/ernie/unsloth_ERNIE-4.5-VL-28B-A3B-PT.yaml @@ -1,6 +1,7 @@ # Model defaults for unsloth/ERNIE-4.5-VL-28B-A3B-PT # Based on ERNIE_4_5_VL_28B_A3B_PT_Vision.ipynb # Also applies to: unsloth/ERNIE-4.5-VL-28B-A3B-PT +# added inference parameters from unsloth notebook training: max_seq_length: 2048 @@ -45,3 +46,7 @@ logging: tensorboard_dir: "runs" log_frequency: 10 +inference: + temperature: 1.5 + min_p: 0.1 + diff --git a/studio/backend/assets/configs/model_defaults/gemma/unsloth_codegemma-7b-bnb-4bit.yaml b/studio/backend/assets/configs/model_defaults/gemma/unsloth_codegemma-7b-bnb-4bit.yaml index 02d0428f94..c6bef5077e 100644 --- a/studio/backend/assets/configs/model_defaults/gemma/unsloth_codegemma-7b-bnb-4bit.yaml +++ b/studio/backend/assets/configs/model_defaults/gemma/unsloth_codegemma-7b-bnb-4bit.yaml @@ -1,6 +1,7 @@ # Model defaults for unsloth/codegemma-7b-bnb-4bit # Based on CodeGemma_(7B)-Conversational.ipynb # Also applies to: unsloth/codegemma-7b, google/codegemma-7b +# added inference parameters from Ollama training: max_seq_length: 4096 @@ -44,3 +45,7 @@ logging: enable_tensorboard: false tensorboard_dir: "runs" log_frequency: 10 + +inference: + temperature: 0 + top_p: 0.9 diff --git a/studio/backend/assets/configs/model_defaults/gemma/unsloth_functiongemma-270m-it.yaml b/studio/backend/assets/configs/model_defaults/gemma/unsloth_functiongemma-270m-it.yaml index 283ab50754..d67877d167 100644 --- a/studio/backend/assets/configs/model_defaults/gemma/unsloth_functiongemma-270m-it.yaml +++ b/studio/backend/assets/configs/model_defaults/gemma/unsloth_functiongemma-270m-it.yaml @@ -1,6 +1,7 @@ # Model defaults for unsloth/functiongemma-270m-it # Based on FunctionGemma_(270M).ipynb # Also applies to: unsloth/functiongemma-270m-it-unsloth-bnb-4bit, google/functiongemma-270m-it, unsloth/functiongemma-270m-it-unsloth-bnb-4bit +# added inference parameters from unsloth guides training: max_seq_length: 4096 @@ -45,3 +46,9 @@ logging: tensorboard_dir: "runs" log_frequency: 10 +inference: + temperature: 1.0 + top_k: 64 + top_p: 0.95 + min_p: 0.0 + diff --git a/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3-270m-it.yaml b/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3-270m-it.yaml index eab0257381..d4108fefb0 100644 --- a/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3-270m-it.yaml +++ b/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3-270m-it.yaml @@ -1,6 +1,7 @@ # Model defaults for unsloth/gemma-3-270m-it # Based on Gemma3_(270M).ipynb # Also applies to: unsloth/gemma-3-270m-it-unsloth-bnb-4bit, google/gemma-3-270m-it, unsloth/gemma-3-270m-it-bnb-4bit +# added inference parameters from unsloth guides training: max_seq_length: 2048 @@ -45,3 +46,9 @@ logging: tensorboard_dir: "runs" log_frequency: 10 +inference: + temperature: 1.0 + top_k: 64 + top_p: 0.95 + min_p: 0.0 + diff --git a/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3-27b-it.yaml b/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3-27b-it.yaml index c8fc2a8269..0f213edb70 100644 --- a/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3-27b-it.yaml +++ b/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3-27b-it.yaml @@ -1,6 +1,7 @@ # Model defaults for unsloth/gemma-3-27b-it # Based on Gemma3_(27B)_A100-Conversational.ipynb # Also applies to: unsloth/gemma-3-27b-it-unsloth-bnb-4bit, google/gemma-3-27b-it, unsloth/gemma-3-27b-it-bnb-4bit +# added inference parameters from unsloth guides training: max_seq_length: 2048 @@ -39,3 +40,9 @@ logging: tensorboard_dir: "runs" log_frequency: 10 +inference: + temperature: 1.0 + top_k: 64 + top_p: 0.95 + min_p: 0.0 + diff --git a/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3-4b-it.yaml b/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3-4b-it.yaml index a9535537b4..d7da7a2873 100644 --- a/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3-4b-it.yaml +++ b/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3-4b-it.yaml @@ -1,6 +1,7 @@ # Model defaults for unsloth/gemma-3-4b-it # Based on Gemma3_(4B).ipynb # Also applies to: unsloth/gemma-3-4b-it-unsloth-bnb-4bit, google/gemma-3-4b-it, unsloth/gemma-3-4b-it-bnb-4bit +# added inference parameters from unsloth guides training: max_seq_length: 2048 @@ -39,3 +40,9 @@ logging: tensorboard_dir: "runs" log_frequency: 10 +inference: + temperature: 1.0 + top_k: 64 + top_p: 0.95 + min_p: 0.0 + diff --git a/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3-4b-pt.yaml b/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3-4b-pt.yaml index 80b85bb836..b0833e6e30 100644 --- a/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3-4b-pt.yaml +++ b/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3-4b-pt.yaml @@ -1,6 +1,7 @@ # Model defaults for unsloth/gemma-3-4b-pt # Based on Gemma3_(4B)-Vision.ipynb # Also applies to: unsloth/gemma-3-4b-pt-unsloth-bnb-4bit, google/gemma-3-4b-pt, unsloth/gemma-3-4b-pt-bnb-4bit +# added inference parameters from unsloth guides training: max_seq_length: 2048 @@ -39,3 +40,9 @@ logging: tensorboard_dir: "runs" log_frequency: 10 +inference: + temperature: 1.0 + top_k: 64 + top_p: 0.95 + min_p: 0.0 + diff --git a/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3n-E4B-it.yaml b/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3n-E4B-it.yaml index 1aeb4f59f8..7fe1fed620 100644 --- a/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3n-E4B-it.yaml +++ b/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3n-E4B-it.yaml @@ -1,6 +1,7 @@ # Model defaults for unsloth/gemma-3n-E4B-it # Based on Gemma3N_(4B)-Conversational.ipynb # Also applies to: unsloth/gemma-3n-E4B-it-unsloth-bnb-4bit, google/gemma-3n-E4B-it, unsloth/gemma-3n-E4B-it-unsloth-bnb-4bit +# added inference parameters from unsloth guides training: max_seq_length: 1024 @@ -39,3 +40,9 @@ logging: tensorboard_dir: "runs" log_frequency: 10 +inference: + temperature: 1.0 + top_k: 64 + top_p: 0.95 + min_p: 0.0 + diff --git a/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3n-E4B.yaml b/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3n-E4B.yaml index 47cbc32782..3b1fab2a60 100644 --- a/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3n-E4B.yaml +++ b/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3n-E4B.yaml @@ -1,6 +1,7 @@ # Model defaults for unsloth/gemma-3n-E4B # Based on Gemma3N_(4B)-Vision.ipynb # Also applies to: unsloth/gemma-3n-E4B-unsloth-bnb-4bit, google/gemma-3n-E4B +# added inference parameters from unsloth guides training: max_seq_length: 2048 @@ -39,3 +40,9 @@ logging: tensorboard_dir: "runs" log_frequency: 10 +inference: + temperature: 1.0 + top_k: 64 + top_p: 0.95 + min_p: 0.0 + diff --git a/studio/backend/assets/configs/model_defaults/gpt-oss/unsloth_gpt-oss-120b.yaml b/studio/backend/assets/configs/model_defaults/gpt-oss/unsloth_gpt-oss-120b.yaml index fc7dedc5fd..93a8d8b274 100644 --- a/studio/backend/assets/configs/model_defaults/gpt-oss/unsloth_gpt-oss-120b.yaml +++ b/studio/backend/assets/configs/model_defaults/gpt-oss/unsloth_gpt-oss-120b.yaml @@ -1,6 +1,7 @@ # Model defaults for unsloth/gpt-oss-120b # Based on gpt-oss-(120B)_A100-Fine-tuning.ipynb # Also applies to: openai/gpt-oss-120b, unsloth/gpt-oss-120b-unsloth-bnb-4bit +# added inference parameters from unsloth guides training: max_seq_length: 4096 @@ -45,3 +46,8 @@ logging: tensorboard_dir: "runs" log_frequency: 10 +inference: + temperature: 1.0 + top_p: 1.0 + top_k: 0 + diff --git a/studio/backend/assets/configs/model_defaults/gpt-oss/unsloth_gpt-oss-20b.yaml b/studio/backend/assets/configs/model_defaults/gpt-oss/unsloth_gpt-oss-20b.yaml index d98dd28700..18f8f7bf47 100644 --- a/studio/backend/assets/configs/model_defaults/gpt-oss/unsloth_gpt-oss-20b.yaml +++ b/studio/backend/assets/configs/model_defaults/gpt-oss/unsloth_gpt-oss-20b.yaml @@ -1,6 +1,7 @@ # Model defaults for unsloth/gpt-oss-20b # Based on gpt-oss-(20B)-Fine-tuning.ipynb # Also applies to: openai/gpt-oss-20b, unsloth/gpt-oss-20b-unsloth-bnb-4bit, unsloth/gpt-oss-20b-BF16 +# added inference parameters from unsloth guides training: max_seq_length: 1024 @@ -45,3 +46,8 @@ logging: tensorboard_dir: "runs" log_frequency: 10 +inference: + temperature: 1.0 + top_p: 1.0 + top_k: 0 + diff --git a/studio/backend/assets/configs/model_defaults/granite/unsloth_granite-4.0-350m-unsloth-bnb-4bit.yaml b/studio/backend/assets/configs/model_defaults/granite/unsloth_granite-4.0-350m-unsloth-bnb-4bit.yaml index 718ec156f8..9b91ec531a 100644 --- a/studio/backend/assets/configs/model_defaults/granite/unsloth_granite-4.0-350m-unsloth-bnb-4bit.yaml +++ b/studio/backend/assets/configs/model_defaults/granite/unsloth_granite-4.0-350m-unsloth-bnb-4bit.yaml @@ -1,6 +1,7 @@ # Model defaults for unsloth/granite-4.0-350m # Based on Granite4.0_350M.ipynb # Also applies to: ibm-granite/granite-4.0-350m, unsloth/granite-4.0-350m-bnb-4bit +# added inference parameters from unsloth guides training: max_seq_length: 2048 @@ -47,3 +48,8 @@ logging: tensorboard_dir: "runs" log_frequency: 10 +inference: + temperature: 0.0 + top_p: 1.0 + top_k: 0 + diff --git a/studio/backend/assets/configs/model_defaults/granite/unsloth_granite-4.0-h-micro.yaml b/studio/backend/assets/configs/model_defaults/granite/unsloth_granite-4.0-h-micro.yaml index 630366faee..e6603b0cea 100644 --- a/studio/backend/assets/configs/model_defaults/granite/unsloth_granite-4.0-h-micro.yaml +++ b/studio/backend/assets/configs/model_defaults/granite/unsloth_granite-4.0-h-micro.yaml @@ -1,6 +1,7 @@ # Model defaults for unsloth/granite-4.0-h-micro # Based on Granite4.0.ipynb # Also applies to: ibm-granite/granite-4.0-h-micro, unsloth/granite-4.0-h-micro-bnb-4bit, unsloth/granite-4.0-h-micro-unsloth-bnb-4bit +# added inference parameters from unsloth guides training: max_seq_length: 2048 @@ -47,3 +48,8 @@ logging: tensorboard_dir: "runs" log_frequency: 10 +inference: + temperature: 0.0 + top_p: 1.0 + top_k: 0 + diff --git a/studio/backend/assets/configs/model_defaults/llama/unsloth_Llama-3.2-11B-Vision-Instruct.yaml b/studio/backend/assets/configs/model_defaults/llama/unsloth_Llama-3.2-11B-Vision-Instruct.yaml index 9147c93586..640ab85b14 100644 --- a/studio/backend/assets/configs/model_defaults/llama/unsloth_Llama-3.2-11B-Vision-Instruct.yaml +++ b/studio/backend/assets/configs/model_defaults/llama/unsloth_Llama-3.2-11B-Vision-Instruct.yaml @@ -1,6 +1,7 @@ # Model defaults for unsloth/Llama-3.2-11B-Vision-Instruct # Based on Llama3.2_(11B)-Vision.ipynb # Also applies to: unsloth/Llama-3.2-11B-Vision-Instruct-unsloth-bnb-4bit, meta-llama/Llama-3.2-11B-Vision-Instruct, unsloth/Llama-3.2-11B-Vision-Instruct-bnb-4bit +# added inference parameters from unsloth notebook training: max_seq_length: 2048 @@ -39,3 +40,7 @@ logging: tensorboard_dir: "runs" log_frequency: 10 +inference: + temperature: 1.5 + min_p: 0.1 + diff --git a/studio/backend/assets/configs/model_defaults/llama/unsloth_Llama-3.2-3B-Instruct.yaml b/studio/backend/assets/configs/model_defaults/llama/unsloth_Llama-3.2-3B-Instruct.yaml index 217dbd50db..18e9428e5e 100644 --- a/studio/backend/assets/configs/model_defaults/llama/unsloth_Llama-3.2-3B-Instruct.yaml +++ b/studio/backend/assets/configs/model_defaults/llama/unsloth_Llama-3.2-3B-Instruct.yaml @@ -1,6 +1,7 @@ # Model defaults for unsloth/Llama-3.2-3B-Instruct # Based on Llama3.2_(1B_and_3B)-Conversational.ipynb # Also applies to: unsloth/Llama-3.2-3B-Instruct-unsloth-bnb-4bit, meta-llama/Llama-3.2-3B-Instruct, unsloth/Llama-3.2-3B-Instruct-bnb-4bit, RedHatAI/Llama-3.2-3B-Instruct-FP8, unsloth/Llama-3.2-3B-Instruct-FP8-Block, unsloth/Llama-3.2-3B-Instruct-FP8-Dynamic +# added inference parameters from unsloth notebook training: max_seq_length: 2048 @@ -45,3 +46,7 @@ logging: tensorboard_dir: "runs" log_frequency: 10 +inference: + temperature: 1.5 + min_p: 0.1 + diff --git a/studio/backend/assets/configs/model_defaults/llama/unsloth_Llama-3.3-70B-Instruct.yaml b/studio/backend/assets/configs/model_defaults/llama/unsloth_Llama-3.3-70B-Instruct.yaml index 166d43a788..8bd5c2f4dc 100644 --- a/studio/backend/assets/configs/model_defaults/llama/unsloth_Llama-3.3-70B-Instruct.yaml +++ b/studio/backend/assets/configs/model_defaults/llama/unsloth_Llama-3.3-70B-Instruct.yaml @@ -1,6 +1,7 @@ # Model defaults for unsloth/Llama-3.3-70B-Instruct # Based on Llama3.3_(70B)_A100-Conversational.ipynb # Also applies to: unsloth/Llama-3.3-70B-Instruct-unsloth-bnb-4bit, meta-llama/Llama-3.3-70B-Instruct, unsloth/Llama-3.3-70B-Instruct-bnb-4bit, RedHatAI/Llama-3.3-70B-Instruct-FP8, unsloth/Llama-3.3-70B-Instruct-FP8-Block, unsloth/Llama-3.3-70B-Instruct-FP8-Dynamic +# added inference parameters from unsloth notebook training: max_seq_length: 2048 @@ -45,3 +46,7 @@ logging: tensorboard_dir: "runs" log_frequency: 10 +inference: + temperature: 1.5 + min_p: 0.1 + diff --git a/studio/backend/assets/configs/model_defaults/llasa/unsloth_Llasa-3B.yaml b/studio/backend/assets/configs/model_defaults/llasa/unsloth_Llasa-3B.yaml index 79e5c9d692..627adfe491 100644 --- a/studio/backend/assets/configs/model_defaults/llasa/unsloth_Llasa-3B.yaml +++ b/studio/backend/assets/configs/model_defaults/llasa/unsloth_Llasa-3B.yaml @@ -1,6 +1,7 @@ # Model defaults for unsloth/Llasa-3B # Based on Llasa_TTS_(3B).ipynb and Llasa_TTS_(1B).ipynb # Also applies to: HKUSTAudio/Llasa-1B +# added inference parameters from unsloth notebook training: max_seq_length: 2048 @@ -40,3 +41,7 @@ logging: tensorboard_dir: "runs" log_frequency: 10 +inference: + temperature: 1.2 + top_p: 1.2 + diff --git a/studio/backend/assets/configs/model_defaults/mistral/unsloth_Magistral-Small-2509-unsloth-bnb-4bit.yaml b/studio/backend/assets/configs/model_defaults/mistral/unsloth_Magistral-Small-2509-unsloth-bnb-4bit.yaml index bb605451a5..9af6fbd3a0 100644 --- a/studio/backend/assets/configs/model_defaults/mistral/unsloth_Magistral-Small-2509-unsloth-bnb-4bit.yaml +++ b/studio/backend/assets/configs/model_defaults/mistral/unsloth_Magistral-Small-2509-unsloth-bnb-4bit.yaml @@ -1,6 +1,7 @@ # Model defaults for unsloth/Magistral-Small-2509 # Based on Magistral_(24B)-Reasoning-Conversational.ipynb # Also applies to: mistralai/Magistral-Small-2509, unsloth/Magistral-Small-2509-bnb-4bit +# added inference parameters from unsloth guides training: max_seq_length: 2048 @@ -45,3 +46,8 @@ logging: tensorboard_dir: "runs" log_frequency: 10 +inference: + temperature: 0.7 + min_p: 0.01 + top_p: 0.95 + diff --git a/studio/backend/assets/configs/model_defaults/mistral/unsloth_Ministral-3-3B-Instruct-2512.yaml b/studio/backend/assets/configs/model_defaults/mistral/unsloth_Ministral-3-3B-Instruct-2512.yaml index b5fd0f5713..d701390da9 100644 --- a/studio/backend/assets/configs/model_defaults/mistral/unsloth_Ministral-3-3B-Instruct-2512.yaml +++ b/studio/backend/assets/configs/model_defaults/mistral/unsloth_Ministral-3-3B-Instruct-2512.yaml @@ -1,6 +1,7 @@ # Model defaults for unsloth/Ministral-3-3B-Instruct-2512 # Based on Ministral_3_VL_(3B)_Vision.ipynb # Also applies to: unsloth/Ministral-3-3B-Instruct-2512 +# added inference parameters from unsloth guides training: max_seq_length: 2048 @@ -45,3 +46,7 @@ logging: tensorboard_dir: "runs" log_frequency: 10 +inference: + temperature: 0.15 + top_p: default + diff --git a/studio/backend/assets/configs/model_defaults/mistral/unsloth_Pixtral-12B-2409.yaml b/studio/backend/assets/configs/model_defaults/mistral/unsloth_Pixtral-12B-2409.yaml index 7945f1944c..9cb15ea2ff 100644 --- a/studio/backend/assets/configs/model_defaults/mistral/unsloth_Pixtral-12B-2409.yaml +++ b/studio/backend/assets/configs/model_defaults/mistral/unsloth_Pixtral-12B-2409.yaml @@ -1,6 +1,7 @@ # Model defaults for unsloth/Pixtral-12B-2409 # Based on Pixtral_(12B)-Vision.ipynb # Also applies to: unsloth/Pixtral-12B-2409-unsloth-bnb-4bit, mistralai/Pixtral-12B-2409, unsloth/Pixtral-12B-2409-bnb-4bit +# added inference parameters from unsloth notebook training: max_seq_length: 2048 @@ -39,3 +40,7 @@ logging: tensorboard_dir: "runs" log_frequency: 10 +inference: + temperature: 1.5 + min_p: 0.1 + diff --git a/studio/backend/assets/configs/model_defaults/other/OuteAI_Llama-OuteTTS-1.0-1B.yaml b/studio/backend/assets/configs/model_defaults/other/OuteAI_Llama-OuteTTS-1.0-1B.yaml index 67cc14b38d..fe0940ddc4 100644 --- a/studio/backend/assets/configs/model_defaults/other/OuteAI_Llama-OuteTTS-1.0-1B.yaml +++ b/studio/backend/assets/configs/model_defaults/other/OuteAI_Llama-OuteTTS-1.0-1B.yaml @@ -1,6 +1,7 @@ # Model defaults for OuteAI/Llama-OuteTTS-1.0-1B # Based on Oute_TTS_(1B).ipynb # Also applies to: OuteAI/Llama-OuteTTS-1.0-1B +# added inference parameters from unsloth notebook training: max_seq_length: 2048 @@ -40,3 +41,9 @@ logging: tensorboard_dir: "runs" log_frequency: 10 +inference: + temperature: 0.4 + top_k: 40 + top_p: 0.9 + min_p: 0.05 + diff --git a/studio/backend/assets/configs/model_defaults/other/Spark-TTS-0.5B_LLM.yaml b/studio/backend/assets/configs/model_defaults/other/Spark-TTS-0.5B_LLM.yaml index 0c08015a7f..02fb108b95 100644 --- a/studio/backend/assets/configs/model_defaults/other/Spark-TTS-0.5B_LLM.yaml +++ b/studio/backend/assets/configs/model_defaults/other/Spark-TTS-0.5B_LLM.yaml @@ -1,6 +1,7 @@ # Model defaults for Spark-TTS-0.5B/LLM # Based on Spark_TTS_(0_5B).ipynb # Also applies to: Spark-TTS-0.5B/LLM +# added inference parameters from unsloth notebook training: max_seq_length: 2048 @@ -45,3 +46,8 @@ logging: tensorboard_dir: "runs" log_frequency: 10 +inference: + temperature: 0.8 + top_k: 50 + top_p: 1.0 + diff --git a/studio/backend/assets/configs/model_defaults/other/unsloth_LFM2-1.2B.yaml b/studio/backend/assets/configs/model_defaults/other/unsloth_LFM2-1.2B.yaml index f51d88b427..2eb6246c23 100644 --- a/studio/backend/assets/configs/model_defaults/other/unsloth_LFM2-1.2B.yaml +++ b/studio/backend/assets/configs/model_defaults/other/unsloth_LFM2-1.2B.yaml @@ -1,6 +1,7 @@ # Model defaults for unsloth/LFM2-1.2B # Based on Liquid_LFM2_(1.2B)-Conversational.ipynb # Also applies to: unsloth/LFM2-1.2B +# added inference parameters from unsloth notebook training: max_seq_length: 2048 @@ -39,3 +40,7 @@ logging: tensorboard_dir: "runs" log_frequency: 10 +inference: + temperature: 0.3 + min_p: 0.15 + diff --git a/studio/backend/assets/configs/model_defaults/other/unsloth_Nemotron-3-Nano-30B-A3B.yaml b/studio/backend/assets/configs/model_defaults/other/unsloth_Nemotron-3-Nano-30B-A3B.yaml index f40a2fd620..0b1b60feac 100644 --- a/studio/backend/assets/configs/model_defaults/other/unsloth_Nemotron-3-Nano-30B-A3B.yaml +++ b/studio/backend/assets/configs/model_defaults/other/unsloth_Nemotron-3-Nano-30B-A3B.yaml @@ -1,6 +1,7 @@ # Model defaults for unsloth/Nemotron-3-Nano-30B-A3B # Based on Nemotron-3-Nano-30B-A3B_A100.ipynb # Also applies to: unsloth/Nemotron-3-Nano-30B-A3B +# added inference parameters from unsloth guides training: max_seq_length: 2048 @@ -47,3 +48,7 @@ logging: tensorboard_dir: "runs" log_frequency: 10 +inference: + temperature: 1.0 + top_p: 1.0 + diff --git a/studio/backend/assets/configs/model_defaults/other/unsloth_PaddleOCR-VL.yaml b/studio/backend/assets/configs/model_defaults/other/unsloth_PaddleOCR-VL.yaml index 370b17c0a3..916333c174 100644 --- a/studio/backend/assets/configs/model_defaults/other/unsloth_PaddleOCR-VL.yaml +++ b/studio/backend/assets/configs/model_defaults/other/unsloth_PaddleOCR-VL.yaml @@ -1,6 +1,7 @@ # Model defaults for unsloth/PaddleOCR-VL # Based on Paddle_OCR_(1B)_Vision.ipynb # Also applies to: unsloth/PaddleOCR-VL +# added inference parameters from unsloth notebook training: max_seq_length: 2048 @@ -45,3 +46,7 @@ logging: tensorboard_dir: "runs" log_frequency: 10 +inference: + temperature: 1.5 + min_p: 0.1 + diff --git a/studio/backend/assets/configs/model_defaults/other/unsloth_orpheus-3b-0.1-ft.yaml b/studio/backend/assets/configs/model_defaults/other/unsloth_orpheus-3b-0.1-ft.yaml index 3eac4fdde5..3079b407df 100644 --- a/studio/backend/assets/configs/model_defaults/other/unsloth_orpheus-3b-0.1-ft.yaml +++ b/studio/backend/assets/configs/model_defaults/other/unsloth_orpheus-3b-0.1-ft.yaml @@ -1,6 +1,7 @@ # Model defaults for unsloth/orpheus-3b-0.1-ft # Based on Orpheus_(3B)-TTS.ipynb # Also applies to: unsloth/orpheus-3b-0.1-ft-unsloth-bnb-4bit, canopylabs/orpheus-3b-0.1-ft, unsloth/orpheus-3b-0.1-ft-bnb-4bit +# added inference parameters from unsloth notebook training: max_seq_length: 2048 @@ -45,3 +46,7 @@ logging: tensorboard_dir: "runs" log_frequency: 10 +inference: + temperature: 0.6 + top_p: 0.95 + diff --git a/studio/backend/assets/configs/model_defaults/phi/unsloth_Phi-4.yaml b/studio/backend/assets/configs/model_defaults/phi/unsloth_Phi-4.yaml index 9746e01374..b13fa9a0b7 100644 --- a/studio/backend/assets/configs/model_defaults/phi/unsloth_Phi-4.yaml +++ b/studio/backend/assets/configs/model_defaults/phi/unsloth_Phi-4.yaml @@ -1,6 +1,7 @@ # Model defaults for unsloth/Phi-4 # Based on Phi_4-Conversational.ipynb # Also applies to: unsloth/phi-4-unsloth-bnb-4bit, microsoft/phi-4, unsloth/phi-4-bnb-4bit +# added inference parameters from unsloth guides training: max_seq_length: 2048 @@ -45,3 +46,7 @@ logging: tensorboard_dir: "runs" log_frequency: 10 +inference: + temperature: 0.8 + top_p: 0.95 + diff --git a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2-VL-7B-Instruct.yaml b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2-VL-7B-Instruct.yaml index 7394967628..b83ca586f5 100644 --- a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2-VL-7B-Instruct.yaml +++ b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2-VL-7B-Instruct.yaml @@ -1,6 +1,7 @@ # Model defaults for unsloth/Qwen2-VL-7B-Instruct # Based on Qwen2_VL_(7B)-Vision.ipynb # Also applies to: unsloth/Qwen2-VL-7B-Instruct-unsloth-bnb-4bit, Qwen/Qwen2-VL-7B-Instruct, unsloth/Qwen2-VL-7B-Instruct-bnb-4bit +# added inference parameters from unsloth notebook training: max_seq_length: 2048 @@ -39,3 +40,7 @@ logging: tensorboard_dir: "runs" log_frequency: 10 +inference: + temperature: 1.5 + min_p: 0.1 + diff --git a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2.5-Coder-14B-Instruct.yaml b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2.5-Coder-14B-Instruct.yaml index 3b45853b6c..f0a27ac90c 100644 --- a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2.5-Coder-14B-Instruct.yaml +++ b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2.5-Coder-14B-Instruct.yaml @@ -1,6 +1,7 @@ # Model defaults for unsloth/Qwen2.5-Coder-14B-Instruct # Based on Qwen2.5_Coder_(14B)-Conversational.ipynb # Also applies to: unsloth/Qwen2.5-Coder-14B-Instruct-bnb-4bit, Qwen/Qwen2.5-Coder-14B-Instruct +# added inference parameters from unsloth notebook training: max_seq_length: 2048 @@ -45,3 +46,7 @@ logging: tensorboard_dir: "runs" log_frequency: 10 +inference: + temperature: 1.5 + min_p: 0.1 + diff --git a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2.5-VL-7B-Instruct-bnb-4bit.yaml b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2.5-VL-7B-Instruct-bnb-4bit.yaml index daa92cf5ed..930f352dd6 100644 --- a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2.5-VL-7B-Instruct-bnb-4bit.yaml +++ b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2.5-VL-7B-Instruct-bnb-4bit.yaml @@ -1,6 +1,7 @@ # Model defaults for unsloth/Qwen2.5-VL-7B-Instruct-bnb-4bit # Based on Qwen2.5_VL_(7B)-Vision.ipynb # Also applies to: unsloth/Qwen2.5-VL-7B-Instruct, Qwen/Qwen2.5-VL-7B-Instruct, unsloth/Qwen2.5-VL-7B-Instruct-unsloth-bnb-4bit +# added inference parameters from unsloth notebook training: max_seq_length: 2048 @@ -39,3 +40,7 @@ logging: tensorboard_dir: "runs" log_frequency: 10 +inference: + temperature: 1.5 + min_p: 0.1 + diff --git a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-0.6B.yaml b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-0.6B.yaml index 153e0d13e1..bf19fad0bd 100644 --- a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-0.6B.yaml +++ b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-0.6B.yaml @@ -1,6 +1,7 @@ # Model defaults for unsloth/Qwen3-0.6B # Based on Qwen3_(0_6B)-Phone_Deployment.ipynb # Also applies to: unsloth/Qwen3-0.6B-unsloth-bnb-4bit, Qwen/Qwen3-0.6B, unsloth/Qwen3-0.6B-bnb-4bit, Qwen/Qwen3-0.6B-FP8, unsloth/Qwen3-0.6B-FP8 +# added inference parameters from Ollama training: max_seq_length: 1024 @@ -45,3 +46,8 @@ logging: tensorboard_dir: "runs" log_frequency: 10 +inference: + temperature: 0.6 + top_k: 20 + top_p: 0.95 + diff --git a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-14B-Base-unsloth-bnb-4bit.yaml b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-14B-Base-unsloth-bnb-4bit.yaml index 76dd3de761..fe519f49a7 100644 --- a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-14B-Base-unsloth-bnb-4bit.yaml +++ b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-14B-Base-unsloth-bnb-4bit.yaml @@ -1,6 +1,7 @@ # Model defaults for unsloth/Qwen3-14B-Base # Based on Qwen3_(14B)-Alpaca.ipynb # Also applies to: unsloth/Qwen3-14B-Base, Qwen/Qwen3-14B-Base, unsloth/Qwen3-14B-Base-bnb-4bit +# added inference parameters from Ollama training: max_seq_length: 2048 @@ -45,3 +46,8 @@ logging: tensorboard_dir: "runs" log_frequency: 10 +inference: + temperature: 0.6 + top_k: 20 + top_p: 0.95 + diff --git a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-14B.yaml b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-14B.yaml index 51bcfc5501..662b96c5fe 100644 --- a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-14B.yaml +++ b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-14B.yaml @@ -1,6 +1,7 @@ # Model defaults for unsloth/Qwen3-14B # Based on Qwen3_(14B).ipynb # Also applies to: unsloth/Qwen3-14B-unsloth-bnb-4bit, Qwen/Qwen3-14B, unsloth/Qwen3-14B-bnb-4bit, Qwen/Qwen3-14B-FP8, unsloth/Qwen3-14B-FP8 +# added inference parameters from Ollama training: max_seq_length: 2048 @@ -45,3 +46,8 @@ logging: tensorboard_dir: "runs" log_frequency: 10 +inference: + temperature: 0.6 + top_k: 20 + top_p: 0.95 + diff --git a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-32B.yaml b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-32B.yaml index 76098e80f5..83ac11efda 100644 --- a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-32B.yaml +++ b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-32B.yaml @@ -1,6 +1,7 @@ # Model defaults for unsloth/Qwen3-32B # Based on Qwen3_(32B)_A100-Reasoning-Conversational.ipynb # Also applies to: unsloth/Qwen3-32B-unsloth-bnb-4bit, Qwen/Qwen3-32B, unsloth/Qwen3-32B-bnb-4bit, Qwen/Qwen3-32B-FP8, unsloth/Qwen3-32B-FP8 +# added inference parameters from Ollama training: max_seq_length: 2048 @@ -45,3 +46,8 @@ logging: tensorboard_dir: "runs" log_frequency: 10 +inference: + temperature: 0.6 + top_k: 20 + top_p: 0.95 + diff --git a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-4B-Instruct-2507.yaml b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-4B-Instruct-2507.yaml index 1a1efaf975..7178b86e7a 100644 --- a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-4B-Instruct-2507.yaml +++ b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-4B-Instruct-2507.yaml @@ -1,6 +1,7 @@ # Model defaults for unsloth/Qwen3-4B-Instruct-2507 # Based on Qwen3_(4B)-Instruct.ipynb # Also applies to: unsloth/Qwen3-4B-Instruct-2507-unsloth-bnb-4bit, Qwen/Qwen3-4B-Instruct-2507, unsloth/Qwen3-4B-Instruct-2507-bnb-4bit, Qwen/Qwen3-4B-Instruct-2507-FP8, unsloth/Qwen3-4B-Instruct-2507-FP8 +# added inference parameters from unsloth guides training: max_seq_length: 2048 @@ -45,3 +46,9 @@ logging: tensorboard_dir: "runs" log_frequency: 10 +inference: + temperature: 0.7 + top_p: 0.80 + top_k: 20 + min_p: 0.00 + diff --git a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-4B-Thinking-2507.yaml b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-4B-Thinking-2507.yaml index 5a67224009..0385f1d00d 100644 --- a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-4B-Thinking-2507.yaml +++ b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-4B-Thinking-2507.yaml @@ -1,6 +1,7 @@ # Model defaults for unsloth/Qwen3-4B-Thinking-2507 # Based on Qwen3_(4B)-Thinking.ipynb # Also applies to: unsloth/Qwen3-4B-Thinking-2507-unsloth-bnb-4bit, Qwen/Qwen3-4B-Thinking-2507, unsloth/Qwen3-4B-Thinking-2507-bnb-4bit, Qwen/Qwen3-4B-Thinking-2507-FP8, unsloth/Qwen3-4B-Thinking-2507-FP8 +# added inference parameters from unsloth guides training: max_seq_length: 2048 @@ -45,3 +46,9 @@ logging: tensorboard_dir: "runs" log_frequency: 10 +inference: + temperature: 0.6 + top_p: 0.95 + top_k: 20 + min_p: 0.00 + diff --git a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-VL-8B-Instruct-unsloth-bnb-4bit.yaml b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-VL-8B-Instruct-unsloth-bnb-4bit.yaml index 56b2feb653..8db73bbdc2 100644 --- a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-VL-8B-Instruct-unsloth-bnb-4bit.yaml +++ b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-VL-8B-Instruct-unsloth-bnb-4bit.yaml @@ -1,6 +1,7 @@ # Model defaults for unsloth/Qwen3-VL-8B-Instruct # Based on Qwen3_VL_(8B)-Vision.ipynb # Also applies to: Qwen/Qwen3-VL-8B-Instruct-FP8, unsloth/Qwen3-VL-8B-Instruct-FP8, unsloth/Qwen3-VL-8B-Instruct, Qwen/Qwen3-VL-8B-Instruct, unsloth/Qwen3-VL-8B-Instruct-bnb-4bit +# added inference parameters from unsloth guides training: max_seq_length: 2048 @@ -39,3 +40,8 @@ logging: tensorboard_dir: "runs" log_frequency: 10 +inference: + temperature: 0.7 + top_p: 0.8 + top_k: 20 + diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py index ada8bdd539..b3924c5569 100644 --- a/studio/backend/models/inference.py +++ b/studio/backend/models/inference.py @@ -43,6 +43,7 @@ class LoadResponse(BaseModel): display_name: str = Field(..., description="Display name of the model") is_vision: bool = Field(False, description="Whether model is a vision model") is_lora: bool = Field(False, description="Whether model is a LoRA adapter") + inference: dict = Field(..., description="Inference parameters (temperature, top_p, top_k, min_p)") class UnloadResponse(BaseModel): diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index b3ea1fa1be..ae8fc31a46 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -22,12 +22,14 @@ if str(backend_path) not in sys.path: try: from core.inference import get_inference_backend from utils.models import ModelConfig + from utils.inference import load_inference_config except ImportError: parent_backend = backend_path.parent / "backend" if str(parent_backend) not in sys.path: sys.path.insert(0, str(parent_backend)) from core.inference import get_inference_backend from utils.models import ModelConfig + from utils.inference import load_inference_config from models.inference import ( LoadRequest, @@ -64,6 +66,8 @@ async def load_model(request: LoadRequest): Load a model for inference. The model_path should be a clean identifier from GET /models/list. + Returns inference configuration parameters (temperature, top_p, top_k, min_p) + from the model's YAML config, falling back to default.yaml for missing values. """ try: backend = get_inference_backend() @@ -97,12 +101,16 @@ async def load_model(request: LoadRequest): logger.info(f"Loaded model: {config.identifier}") + # Load inference configuration parameters + inference_config = load_inference_config(config.identifier) + return LoadResponse( status="loaded", model=config.identifier, display_name=config.display_name, is_vision=config.is_vision, is_lora=config.is_lora, + inference=inference_config, ) except HTTPException: diff --git a/studio/backend/utils/inference/__init__.py b/studio/backend/utils/inference/__init__.py new file mode 100644 index 0000000000..660643a436 --- /dev/null +++ b/studio/backend/utils/inference/__init__.py @@ -0,0 +1,7 @@ +""" +Inference utility functions +""" +from utils.inference.inference_config import load_inference_config + +__all__ = ["load_inference_config"] + diff --git a/studio/backend/utils/inference/inference_config.py b/studio/backend/utils/inference/inference_config.py new file mode 100644 index 0000000000..d6de562d33 --- /dev/null +++ b/studio/backend/utils/inference/inference_config.py @@ -0,0 +1,65 @@ +""" +Inference configuration loading utilities. + +This module provides functions to load inference parameters (temperature, top_p, top_k, min_p) +from model YAML configuration files, with fallback to default.yaml. +""" +from pathlib import Path +from typing import Dict, Any +import yaml +import logging + +from utils.models.model_config import load_model_defaults + +logger = logging.getLogger(__name__) + + +def load_inference_config(model_identifier: str) -> Dict[str, Any]: + """ + Load inference configuration parameters for a model. + + This function loads inference parameters (temperature, top_p, top_k, min_p) from the + model's YAML configuration file using the same mapping logic as the /config endpoint. + If a parameter is missing from the model's config, it falls back to the value in + default.yaml. + + Args: + model_identifier: Model identifier (e.g., "unsloth/llama-3-8b-bnb-4bit") + + Returns: + Dictionary containing inference parameters: + { + "temperature": float, + "top_p": float, + "top_k": int, + "min_p": float + } + """ + # Load model defaults to get inference parameters + model_defaults = load_model_defaults(model_identifier) + + # Load default.yaml for fallback values + script_dir = Path(__file__).parent.parent.parent + defaults_dir = script_dir / "assets" / "configs" / "model_defaults" + default_config_path = defaults_dir / "default.yaml" + + default_inference = {} + if default_config_path.exists(): + try: + with open(default_config_path, 'r', encoding='utf-8') as f: + default_config = yaml.safe_load(f) or {} + default_inference = default_config.get("inference", {}) + except Exception as e: + logger.warning(f"Failed to load default.yaml: {e}") + + # Extract inference parameters from model config, fallback to defaults + model_inference = model_defaults.get("inference", {}) + inference_config = { + "temperature": model_inference.get("temperature", default_inference.get("temperature", 0.7)), + "top_p": model_inference.get("top_p", default_inference.get("top_p", 0.95)), + "top_k": model_inference.get("top_k", default_inference.get("top_k", -1)), + "min_p": model_inference.get("min_p", default_inference.get("min_p", 0.01)), + } + + return inference_config +