ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #96 from unslothai/feature/inference-yaml-ordered

Browse source 
Added the inference defaults for models
This commit is contained in:
Roland Tannous 2026-02-15 17:43:31 +04:00 • committed by GitHub
commit 4dbd77786e
42 changed files with 303 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

6
studio/backend/assets/configs/model_defaults/default.yaml
View file

@ -45,3 +45,9 @@ logging:
tensorboard_dir: "runs"
log_frequency: 10
inference:
temperature: 0.7
top_p: 0.95
top_k: -1
min_p: 0.01

5
studio/backend/assets/configs/model_defaults/ernie/unsloth_ERNIE-4.5-VL-28B-A3B-PT.yaml
View file

@ -1,6 +1,7 @@
# Model defaults for unsloth/ERNIE-4.5-VL-28B-A3B-PT
# Based on ERNIE_4_5_VL_28B_A3B_PT_Vision.ipynb
# Also applies to: unsloth/ERNIE-4.5-VL-28B-A3B-PT
# added inference parameters from unsloth notebook
training:
max_seq_length: 2048
@ -45,3 +46,7 @@ logging:
tensorboard_dir: "runs"
log_frequency: 10
inference:
temperature: 1.5
min_p: 0.1

5
studio/backend/assets/configs/model_defaults/gemma/unsloth_codegemma-7b-bnb-4bit.yaml
View file

@ -1,6 +1,7 @@
# Model defaults for unsloth/codegemma-7b-bnb-4bit
# Based on CodeGemma_(7B)-Conversational.ipynb
# Also applies to: unsloth/codegemma-7b, google/codegemma-7b
# added inference parameters from Ollama
training:
max_seq_length: 4096
@ -44,3 +45,7 @@ logging:
enable_tensorboard: false
tensorboard_dir: "runs"
log_frequency: 10
inference:
temperature: 0
top_p: 0.9

7
studio/backend/assets/configs/model_defaults/gemma/unsloth_functiongemma-270m-it.yaml
View file

@ -1,6 +1,7 @@
# Model defaults for unsloth/functiongemma-270m-it
# Based on FunctionGemma_(270M).ipynb
# Also applies to: unsloth/functiongemma-270m-it-unsloth-bnb-4bit, google/functiongemma-270m-it, unsloth/functiongemma-270m-it-unsloth-bnb-4bit
# added inference parameters from unsloth guides
training:
max_seq_length: 4096
@ -45,3 +46,9 @@ logging:
tensorboard_dir: "runs"
log_frequency: 10
inference:
temperature: 1.0
top_k: 64
top_p: 0.95
min_p: 0.0

7
studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3-270m-it.yaml
View file

@ -1,6 +1,7 @@
# Model defaults for unsloth/gemma-3-270m-it
# Based on Gemma3_(270M).ipynb
# Also applies to: unsloth/gemma-3-270m-it-unsloth-bnb-4bit, google/gemma-3-270m-it, unsloth/gemma-3-270m-it-bnb-4bit
# added inference parameters from unsloth guides
training:
max_seq_length: 2048
@ -45,3 +46,9 @@ logging:
tensorboard_dir: "runs"
log_frequency: 10
inference:
temperature: 1.0
top_k: 64
top_p: 0.95
min_p: 0.0

7
studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3-27b-it.yaml
View file

@ -1,6 +1,7 @@
# Model defaults for unsloth/gemma-3-27b-it
# Based on Gemma3_(27B)_A100-Conversational.ipynb
# Also applies to: unsloth/gemma-3-27b-it-unsloth-bnb-4bit, google/gemma-3-27b-it, unsloth/gemma-3-27b-it-bnb-4bit
# added inference parameters from unsloth guides
training:
max_seq_length: 2048
@ -39,3 +40,9 @@ logging:
tensorboard_dir: "runs"
log_frequency: 10
inference:
temperature: 1.0
top_k: 64
top_p: 0.95
min_p: 0.0

7
studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3-4b-it.yaml
View file

@ -1,6 +1,7 @@
# Model defaults for unsloth/gemma-3-4b-it
# Based on Gemma3_(4B).ipynb
# Also applies to: unsloth/gemma-3-4b-it-unsloth-bnb-4bit, google/gemma-3-4b-it, unsloth/gemma-3-4b-it-bnb-4bit
# added inference parameters from unsloth guides
training:
max_seq_length: 2048
@ -39,3 +40,9 @@ logging:
tensorboard_dir: "runs"
log_frequency: 10
inference:
temperature: 1.0
top_k: 64
top_p: 0.95
min_p: 0.0

7
studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3-4b-pt.yaml
View file

@ -1,6 +1,7 @@
# Model defaults for unsloth/gemma-3-4b-pt
# Based on Gemma3_(4B)-Vision.ipynb
# Also applies to: unsloth/gemma-3-4b-pt-unsloth-bnb-4bit, google/gemma-3-4b-pt, unsloth/gemma-3-4b-pt-bnb-4bit
# added inference parameters from unsloth guides
training:
max_seq_length: 2048
@ -39,3 +40,9 @@ logging:
tensorboard_dir: "runs"
log_frequency: 10
inference:
temperature: 1.0
top_k: 64
top_p: 0.95
min_p: 0.0

7
studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3n-E4B-it.yaml
View file

@ -1,6 +1,7 @@
# Model defaults for unsloth/gemma-3n-E4B-it
# Based on Gemma3N_(4B)-Conversational.ipynb
# Also applies to: unsloth/gemma-3n-E4B-it-unsloth-bnb-4bit, google/gemma-3n-E4B-it, unsloth/gemma-3n-E4B-it-unsloth-bnb-4bit
# added inference parameters from unsloth guides
training:
max_seq_length: 1024
@ -39,3 +40,9 @@ logging:
tensorboard_dir: "runs"
log_frequency: 10
inference:
temperature: 1.0
top_k: 64
top_p: 0.95
min_p: 0.0

7
studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3n-E4B.yaml
View file

@ -1,6 +1,7 @@
# Model defaults for unsloth/gemma-3n-E4B
# Based on Gemma3N_(4B)-Vision.ipynb
# Also applies to: unsloth/gemma-3n-E4B-unsloth-bnb-4bit, google/gemma-3n-E4B
# added inference parameters from unsloth guides
training:
max_seq_length: 2048
@ -39,3 +40,9 @@ logging:
tensorboard_dir: "runs"
log_frequency: 10
inference:
temperature: 1.0
top_k: 64
top_p: 0.95
min_p: 0.0

6
studio/backend/assets/configs/model_defaults/gpt-oss/unsloth_gpt-oss-120b.yaml
View file

@ -1,6 +1,7 @@
# Model defaults for unsloth/gpt-oss-120b
# Based on gpt-oss-(120B)_A100-Fine-tuning.ipynb
# Also applies to: openai/gpt-oss-120b, unsloth/gpt-oss-120b-unsloth-bnb-4bit
# added inference parameters from unsloth guides
training:
max_seq_length: 4096
@ -45,3 +46,8 @@ logging:
tensorboard_dir: "runs"
log_frequency: 10
inference:
temperature: 1.0
top_p: 1.0
top_k: 0

6
studio/backend/assets/configs/model_defaults/gpt-oss/unsloth_gpt-oss-20b.yaml
View file

@ -1,6 +1,7 @@
# Model defaults for unsloth/gpt-oss-20b
# Based on gpt-oss-(20B)-Fine-tuning.ipynb
# Also applies to: openai/gpt-oss-20b, unsloth/gpt-oss-20b-unsloth-bnb-4bit, unsloth/gpt-oss-20b-BF16
# added inference parameters from unsloth guides
training:
max_seq_length: 1024
@ -45,3 +46,8 @@ logging:
tensorboard_dir: "runs"
log_frequency: 10
inference:
temperature: 1.0
top_p: 1.0
top_k: 0

6
studio/backend/assets/configs/model_defaults/granite/unsloth_granite-4.0-350m-unsloth-bnb-4bit.yaml
View file

@ -1,6 +1,7 @@
# Model defaults for unsloth/granite-4.0-350m
# Based on Granite4.0_350M.ipynb
# Also applies to: ibm-granite/granite-4.0-350m, unsloth/granite-4.0-350m-bnb-4bit
# added inference parameters from unsloth guides
training:
max_seq_length: 2048
@ -47,3 +48,8 @@ logging:
tensorboard_dir: "runs"
log_frequency: 10
inference:
temperature: 0.0
top_p: 1.0
top_k: 0

6
studio/backend/assets/configs/model_defaults/granite/unsloth_granite-4.0-h-micro.yaml
View file

@ -1,6 +1,7 @@
# Model defaults for unsloth/granite-4.0-h-micro
# Based on Granite4.0.ipynb
# Also applies to: ibm-granite/granite-4.0-h-micro, unsloth/granite-4.0-h-micro-bnb-4bit, unsloth/granite-4.0-h-micro-unsloth-bnb-4bit
# added inference parameters from unsloth guides
training:
max_seq_length: 2048
@ -47,3 +48,8 @@ logging:
tensorboard_dir: "runs"
log_frequency: 10
inference:
temperature: 0.0
top_p: 1.0
top_k: 0

5
studio/backend/assets/configs/model_defaults/llama/unsloth_Llama-3.2-11B-Vision-Instruct.yaml
View file

@ -1,6 +1,7 @@
# Model defaults for unsloth/Llama-3.2-11B-Vision-Instruct
# Based on Llama3.2_(11B)-Vision.ipynb
# Also applies to: unsloth/Llama-3.2-11B-Vision-Instruct-unsloth-bnb-4bit, meta-llama/Llama-3.2-11B-Vision-Instruct, unsloth/Llama-3.2-11B-Vision-Instruct-bnb-4bit
# added inference parameters from unsloth notebook
training:
max_seq_length: 2048
@ -39,3 +40,7 @@ logging:
tensorboard_dir: "runs"
log_frequency: 10
inference:
temperature: 1.5
min_p: 0.1

5
studio/backend/assets/configs/model_defaults/llama/unsloth_Llama-3.2-3B-Instruct.yaml
View file

@ -1,6 +1,7 @@
# Model defaults for unsloth/Llama-3.2-3B-Instruct
# Based on Llama3.2_(1B_and_3B)-Conversational.ipynb
# Also applies to: unsloth/Llama-3.2-3B-Instruct-unsloth-bnb-4bit, meta-llama/Llama-3.2-3B-Instruct, unsloth/Llama-3.2-3B-Instruct-bnb-4bit, RedHatAI/Llama-3.2-3B-Instruct-FP8, unsloth/Llama-3.2-3B-Instruct-FP8-Block, unsloth/Llama-3.2-3B-Instruct-FP8-Dynamic
# added inference parameters from unsloth notebook
training:
max_seq_length: 2048
@ -45,3 +46,7 @@ logging:
tensorboard_dir: "runs"
log_frequency: 10
inference:
temperature: 1.5
min_p: 0.1

5
studio/backend/assets/configs/model_defaults/llama/unsloth_Llama-3.3-70B-Instruct.yaml
View file

@ -1,6 +1,7 @@
# Model defaults for unsloth/Llama-3.3-70B-Instruct
# Based on Llama3.3_(70B)_A100-Conversational.ipynb
# Also applies to: unsloth/Llama-3.3-70B-Instruct-unsloth-bnb-4bit, meta-llama/Llama-3.3-70B-Instruct, unsloth/Llama-3.3-70B-Instruct-bnb-4bit, RedHatAI/Llama-3.3-70B-Instruct-FP8, unsloth/Llama-3.3-70B-Instruct-FP8-Block, unsloth/Llama-3.3-70B-Instruct-FP8-Dynamic
# added inference parameters from unsloth notebook
training:
max_seq_length: 2048
@ -45,3 +46,7 @@ logging:
tensorboard_dir: "runs"
log_frequency: 10
inference:
temperature: 1.5
min_p: 0.1

5
studio/backend/assets/configs/model_defaults/llasa/unsloth_Llasa-3B.yaml
View file

@ -1,6 +1,7 @@
# Model defaults for unsloth/Llasa-3B
# Based on Llasa_TTS_(3B).ipynb and Llasa_TTS_(1B).ipynb
# Also applies to: HKUSTAudio/Llasa-1B
# added inference parameters from unsloth notebook
training:
max_seq_length: 2048
@ -40,3 +41,7 @@ logging:
tensorboard_dir: "runs"
log_frequency: 10
inference:
temperature: 1.2
top_p: 1.2

6
studio/backend/assets/configs/model_defaults/mistral/unsloth_Magistral-Small-2509-unsloth-bnb-4bit.yaml
View file

@ -1,6 +1,7 @@
# Model defaults for unsloth/Magistral-Small-2509
# Based on Magistral_(24B)-Reasoning-Conversational.ipynb
# Also applies to: mistralai/Magistral-Small-2509, unsloth/Magistral-Small-2509-bnb-4bit
# added inference parameters from unsloth guides
training:
max_seq_length: 2048
@ -45,3 +46,8 @@ logging:
tensorboard_dir: "runs"
log_frequency: 10
inference:
temperature: 0.7
min_p: 0.01
top_p: 0.95

5
studio/backend/assets/configs/model_defaults/mistral/unsloth_Ministral-3-3B-Instruct-2512.yaml
View file

@ -1,6 +1,7 @@
# Model defaults for unsloth/Ministral-3-3B-Instruct-2512
# Based on Ministral_3_VL_(3B)_Vision.ipynb
# Also applies to: unsloth/Ministral-3-3B-Instruct-2512
# added inference parameters from unsloth guides
training:
max_seq_length: 2048
@ -45,3 +46,7 @@ logging:
tensorboard_dir: "runs"
log_frequency: 10
inference:
temperature: 0.15
top_p: default

5
studio/backend/assets/configs/model_defaults/mistral/unsloth_Pixtral-12B-2409.yaml
View file

@ -1,6 +1,7 @@
# Model defaults for unsloth/Pixtral-12B-2409
# Based on Pixtral_(12B)-Vision.ipynb
# Also applies to: unsloth/Pixtral-12B-2409-unsloth-bnb-4bit, mistralai/Pixtral-12B-2409, unsloth/Pixtral-12B-2409-bnb-4bit
# added inference parameters from unsloth notebook
training:
max_seq_length: 2048
@ -39,3 +40,7 @@ logging:
tensorboard_dir: "runs"
log_frequency: 10
inference:
temperature: 1.5
min_p: 0.1

7
studio/backend/assets/configs/model_defaults/other/OuteAI_Llama-OuteTTS-1.0-1B.yaml
View file

@ -1,6 +1,7 @@
# Model defaults for OuteAI/Llama-OuteTTS-1.0-1B
# Based on Oute_TTS_(1B).ipynb
# Also applies to: OuteAI/Llama-OuteTTS-1.0-1B
# added inference parameters from unsloth notebook
training:
max_seq_length: 2048
@ -40,3 +41,9 @@ logging:
tensorboard_dir: "runs"
log_frequency: 10
inference:
temperature: 0.4
top_k: 40
top_p: 0.9
min_p: 0.05

6
studio/backend/assets/configs/model_defaults/other/Spark-TTS-0.5B_LLM.yaml
View file

@ -1,6 +1,7 @@
# Model defaults for Spark-TTS-0.5B/LLM
# Based on Spark_TTS_(0_5B).ipynb
# Also applies to: Spark-TTS-0.5B/LLM
# added inference parameters from unsloth notebook
training:
max_seq_length: 2048
@ -45,3 +46,8 @@ logging:
tensorboard_dir: "runs"
log_frequency: 10
inference:
temperature: 0.8
top_k: 50
top_p: 1.0

5
studio/backend/assets/configs/model_defaults/other/unsloth_LFM2-1.2B.yaml
View file

@ -1,6 +1,7 @@
# Model defaults for unsloth/LFM2-1.2B
# Based on Liquid_LFM2_(1.2B)-Conversational.ipynb
# Also applies to: unsloth/LFM2-1.2B
# added inference parameters from unsloth notebook
training:
max_seq_length: 2048
@ -39,3 +40,7 @@ logging:
tensorboard_dir: "runs"
log_frequency: 10
inference:
temperature: 0.3
min_p: 0.15

5
studio/backend/assets/configs/model_defaults/other/unsloth_Nemotron-3-Nano-30B-A3B.yaml
View file

@ -1,6 +1,7 @@
# Model defaults for unsloth/Nemotron-3-Nano-30B-A3B
# Based on Nemotron-3-Nano-30B-A3B_A100.ipynb
# Also applies to: unsloth/Nemotron-3-Nano-30B-A3B
# added inference parameters from unsloth guides
training:
max_seq_length: 2048
@ -47,3 +48,7 @@ logging:
tensorboard_dir: "runs"
log_frequency: 10
inference:
temperature: 1.0
top_p: 1.0

5
studio/backend/assets/configs/model_defaults/other/unsloth_PaddleOCR-VL.yaml
View file

@ -1,6 +1,7 @@
# Model defaults for unsloth/PaddleOCR-VL
# Based on Paddle_OCR_(1B)_Vision.ipynb
# Also applies to: unsloth/PaddleOCR-VL
# added inference parameters from unsloth notebook
training:
max_seq_length: 2048
@ -45,3 +46,7 @@ logging:
tensorboard_dir: "runs"
log_frequency: 10
inference:
temperature: 1.5
min_p: 0.1

5
studio/backend/assets/configs/model_defaults/other/unsloth_orpheus-3b-0.1-ft.yaml
View file

@ -1,6 +1,7 @@
# Model defaults for unsloth/orpheus-3b-0.1-ft
# Based on Orpheus_(3B)-TTS.ipynb
# Also applies to: unsloth/orpheus-3b-0.1-ft-unsloth-bnb-4bit, canopylabs/orpheus-3b-0.1-ft, unsloth/orpheus-3b-0.1-ft-bnb-4bit
# added inference parameters from unsloth notebook
training:
max_seq_length: 2048
@ -45,3 +46,7 @@ logging:
tensorboard_dir: "runs"
log_frequency: 10
inference:
temperature: 0.6
top_p: 0.95

5
studio/backend/assets/configs/model_defaults/phi/unsloth_Phi-4.yaml
View file

@ -1,6 +1,7 @@
# Model defaults for unsloth/Phi-4
# Based on Phi_4-Conversational.ipynb
# Also applies to: unsloth/phi-4-unsloth-bnb-4bit, microsoft/phi-4, unsloth/phi-4-bnb-4bit
# added inference parameters from unsloth guides
training:
max_seq_length: 2048
@ -45,3 +46,7 @@ logging:
tensorboard_dir: "runs"
log_frequency: 10
inference:
temperature: 0.8
top_p: 0.95

5
studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2-VL-7B-Instruct.yaml
View file

@ -1,6 +1,7 @@
# Model defaults for unsloth/Qwen2-VL-7B-Instruct
# Based on Qwen2_VL_(7B)-Vision.ipynb
# Also applies to: unsloth/Qwen2-VL-7B-Instruct-unsloth-bnb-4bit, Qwen/Qwen2-VL-7B-Instruct, unsloth/Qwen2-VL-7B-Instruct-bnb-4bit
# added inference parameters from unsloth notebook
training:
max_seq_length: 2048
@ -39,3 +40,7 @@ logging:
tensorboard_dir: "runs"
log_frequency: 10
inference:
temperature: 1.5
min_p: 0.1

5
studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2.5-Coder-14B-Instruct.yaml
View file

@ -1,6 +1,7 @@
# Model defaults for unsloth/Qwen2.5-Coder-14B-Instruct
# Based on Qwen2.5_Coder_(14B)-Conversational.ipynb
# Also applies to: unsloth/Qwen2.5-Coder-14B-Instruct-bnb-4bit, Qwen/Qwen2.5-Coder-14B-Instruct
# added inference parameters from unsloth notebook
training:
max_seq_length: 2048
@ -45,3 +46,7 @@ logging:
tensorboard_dir: "runs"
log_frequency: 10
inference:
temperature: 1.5
min_p: 0.1

5
studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2.5-VL-7B-Instruct-bnb-4bit.yaml
View file

@ -1,6 +1,7 @@
# Model defaults for unsloth/Qwen2.5-VL-7B-Instruct-bnb-4bit
# Based on Qwen2.5_VL_(7B)-Vision.ipynb
# Also applies to: unsloth/Qwen2.5-VL-7B-Instruct, Qwen/Qwen2.5-VL-7B-Instruct, unsloth/Qwen2.5-VL-7B-Instruct-unsloth-bnb-4bit
# added inference parameters from unsloth notebook
training:
max_seq_length: 2048
@ -39,3 +40,7 @@ logging:
tensorboard_dir: "runs"
log_frequency: 10
inference:
temperature: 1.5
min_p: 0.1

6
studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-0.6B.yaml
View file

@ -1,6 +1,7 @@
# Model defaults for unsloth/Qwen3-0.6B
# Based on Qwen3_(0_6B)-Phone_Deployment.ipynb
# Also applies to: unsloth/Qwen3-0.6B-unsloth-bnb-4bit, Qwen/Qwen3-0.6B, unsloth/Qwen3-0.6B-bnb-4bit, Qwen/Qwen3-0.6B-FP8, unsloth/Qwen3-0.6B-FP8
# added inference parameters from Ollama
training:
max_seq_length: 1024
@ -45,3 +46,8 @@ logging:
tensorboard_dir: "runs"
log_frequency: 10
inference:
temperature: 0.6
top_k: 20
top_p: 0.95

6
studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-14B-Base-unsloth-bnb-4bit.yaml
View file

@ -1,6 +1,7 @@
# Model defaults for unsloth/Qwen3-14B-Base
# Based on Qwen3_(14B)-Alpaca.ipynb
# Also applies to: unsloth/Qwen3-14B-Base, Qwen/Qwen3-14B-Base, unsloth/Qwen3-14B-Base-bnb-4bit
# added inference parameters from Ollama
training:
max_seq_length: 2048
@ -45,3 +46,8 @@ logging:
tensorboard_dir: "runs"
log_frequency: 10
inference:
temperature: 0.6
top_k: 20
top_p: 0.95

6
studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-14B.yaml
View file

@ -1,6 +1,7 @@
# Model defaults for unsloth/Qwen3-14B
# Based on Qwen3_(14B).ipynb
# Also applies to: unsloth/Qwen3-14B-unsloth-bnb-4bit, Qwen/Qwen3-14B, unsloth/Qwen3-14B-bnb-4bit, Qwen/Qwen3-14B-FP8, unsloth/Qwen3-14B-FP8
# added inference parameters from Ollama
training:
max_seq_length: 2048
@ -45,3 +46,8 @@ logging:
tensorboard_dir: "runs"
log_frequency: 10
inference:
temperature: 0.6
top_k: 20
top_p: 0.95

6
studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-32B.yaml
View file

@ -1,6 +1,7 @@
# Model defaults for unsloth/Qwen3-32B
# Based on Qwen3_(32B)_A100-Reasoning-Conversational.ipynb
# Also applies to: unsloth/Qwen3-32B-unsloth-bnb-4bit, Qwen/Qwen3-32B, unsloth/Qwen3-32B-bnb-4bit, Qwen/Qwen3-32B-FP8, unsloth/Qwen3-32B-FP8
# added inference parameters from Ollama
training:
max_seq_length: 2048
@ -45,3 +46,8 @@ logging:
tensorboard_dir: "runs"
log_frequency: 10
inference:
temperature: 0.6
top_k: 20
top_p: 0.95

7
studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-4B-Instruct-2507.yaml
View file

@ -1,6 +1,7 @@
# Model defaults for unsloth/Qwen3-4B-Instruct-2507
# Based on Qwen3_(4B)-Instruct.ipynb
# Also applies to: unsloth/Qwen3-4B-Instruct-2507-unsloth-bnb-4bit, Qwen/Qwen3-4B-Instruct-2507, unsloth/Qwen3-4B-Instruct-2507-bnb-4bit, Qwen/Qwen3-4B-Instruct-2507-FP8, unsloth/Qwen3-4B-Instruct-2507-FP8
# added inference parameters from unsloth guides
training:
max_seq_length: 2048
@ -45,3 +46,9 @@ logging:
tensorboard_dir: "runs"
log_frequency: 10
inference:
temperature: 0.7
top_p: 0.80
top_k: 20
min_p: 0.00

7
studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-4B-Thinking-2507.yaml
View file

@ -1,6 +1,7 @@
# Model defaults for unsloth/Qwen3-4B-Thinking-2507
# Based on Qwen3_(4B)-Thinking.ipynb
# Also applies to: unsloth/Qwen3-4B-Thinking-2507-unsloth-bnb-4bit, Qwen/Qwen3-4B-Thinking-2507, unsloth/Qwen3-4B-Thinking-2507-bnb-4bit, Qwen/Qwen3-4B-Thinking-2507-FP8, unsloth/Qwen3-4B-Thinking-2507-FP8
# added inference parameters from unsloth guides
training:
max_seq_length: 2048
@ -45,3 +46,9 @@ logging:
tensorboard_dir: "runs"
log_frequency: 10
inference:
temperature: 0.6
top_p: 0.95
top_k: 20
min_p: 0.00

6
studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-VL-8B-Instruct-unsloth-bnb-4bit.yaml
View file

@ -1,6 +1,7 @@
# Model defaults for unsloth/Qwen3-VL-8B-Instruct
# Based on Qwen3_VL_(8B)-Vision.ipynb
# Also applies to: Qwen/Qwen3-VL-8B-Instruct-FP8, unsloth/Qwen3-VL-8B-Instruct-FP8, unsloth/Qwen3-VL-8B-Instruct, Qwen/Qwen3-VL-8B-Instruct, unsloth/Qwen3-VL-8B-Instruct-bnb-4bit
# added inference parameters from unsloth guides
training:
max_seq_length: 2048
@ -39,3 +40,8 @@ logging:
tensorboard_dir: "runs"
log_frequency: 10
inference:
temperature: 0.7
top_p: 0.8
top_k: 20

1
studio/backend/models/inference.py
View file

@ -43,6 +43,7 @@ class LoadResponse(BaseModel):
display_name: str = Field(..., description="Display name of the model")
is_vision: bool = Field(False, description="Whether model is a vision model")
is_lora: bool = Field(False, description="Whether model is a LoRA adapter")
inference: dict = Field(..., description="Inference parameters (temperature, top_p, top_k, min_p)")
class UnloadResponse(BaseModel):

8
studio/backend/routes/inference.py
View file

@ -22,12 +22,14 @@ if str(backend_path) not in sys.path:
try:
from core.inference import get_inference_backend
from utils.models import ModelConfig
from utils.inference import load_inference_config
except ImportError:
parent_backend = backend_path.parent / "backend"
if str(parent_backend) not in sys.path:
sys.path.insert(0, str(parent_backend))
from core.inference import get_inference_backend
from utils.models import ModelConfig
from utils.inference import load_inference_config
from models.inference import (
LoadRequest,
@ -64,6 +66,8 @@ async def load_model(request: LoadRequest):
Load a model for inference.
The model_path should be a clean identifier from GET /models/list.
Returns inference configuration parameters (temperature, top_p, top_k, min_p)
from the model's YAML config, falling back to default.yaml for missing values.
"""
try:
backend = get_inference_backend()
@ -97,12 +101,16 @@ async def load_model(request: LoadRequest):
logger.info(f"Loaded model: {config.identifier}")
# Load inference configuration parameters
inference_config = load_inference_config(config.identifier)
return LoadResponse(
status="loaded",
model=config.identifier,
display_name=config.display_name,
is_vision=config.is_vision,
is_lora=config.is_lora,
inference=inference_config,
)
except HTTPException:

7
studio/backend/utils/inference/__init__.py Normal file
View file

@ -0,0 +1,7 @@
"""
Inference utility functions
"""
from utils.inference.inference_config import load_inference_config
__all__ = ["load_inference_config"]

65
studio/backend/utils/inference/inference_config.py Normal file
View file

@ -0,0 +1,65 @@
"""
Inference configuration loading utilities.
This module provides functions to load inference parameters (temperature, top_p, top_k, min_p)
from model YAML configuration files, with fallback to default.yaml.
"""
from pathlib import Path
from typing import Dict, Any
import yaml
import logging
from utils.models.model_config import load_model_defaults
logger = logging.getLogger(__name__)
def load_inference_config(model_identifier: str) -> Dict[str, Any]:
"""
Load inference configuration parameters for a model.
This function loads inference parameters (temperature, top_p, top_k, min_p) from the
model's YAML configuration file using the same mapping logic as the /config endpoint.
If a parameter is missing from the model's config, it falls back to the value in
default.yaml.
Args:
model_identifier: Model identifier (e.g., "unsloth/llama-3-8b-bnb-4bit")
Returns:
Dictionary containing inference parameters:
{
"temperature": float,
"top_p": float,
"top_k": int,
"min_p": float
}
"""
# Load model defaults to get inference parameters
model_defaults = load_model_defaults(model_identifier)
# Load default.yaml for fallback values
script_dir = Path(__file__).parent.parent.parent
defaults_dir = script_dir / "assets" / "configs" / "model_defaults"
default_config_path = defaults_dir / "default.yaml"
default_inference = {}
if default_config_path.exists():
try:
with open(default_config_path, 'r', encoding='utf-8') as f:
default_config = yaml.safe_load(f) or {}
default_inference = default_config.get("inference", {})
except Exception as e:
logger.warning(f"Failed to load default.yaml: {e}")
# Extract inference parameters from model config, fallback to defaults
model_inference = model_defaults.get("inference", {})
inference_config = {
"temperature": model_inference.get("temperature", default_inference.get("temperature", 0.7)),
"top_p": model_inference.get("top_p", default_inference.get("top_p", 0.95)),
"top_k": model_inference.get("top_k", default_inference.get("top_k", -1)),
"min_p": model_inference.get("min_p", default_inference.get("min_p", 0.01)),
}
return inference_config