From ddbbfe52cf98848f9cfed1d5cd67a0e9e0d0c6cd Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 17 Mar 2026 07:58:28 -0700 Subject: [PATCH 001/132] Update pyproject.toml --- pyproject.toml | 636 ++++++++----------------------------------------- 1 file changed, 95 insertions(+), 541 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f7112aa89e..5a75495c56 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,29 @@ classifiers = [ "Topic :: Scientific/Engineering :: Artificial Intelligence", ] dependencies = [ + "unsloth_zoo>=2026.3.4", + "wheel>=0.42.0", + "packaging", + "torch>=2.4.0", + "torchvision", + "numpy", + "tqdm", + "psutil", + "tyro", + "protobuf", + "xformers>=0.0.27.post2 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", + "triton>=3.0.0 ; ('linux' in sys_platform)", + "triton-windows ; (sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "sentencepiece>=0.2.0", + "datasets>=3.4.1,!=4.0.*,!=4.1.0,<4.4.0", + "accelerate>=0.34.1", + "peft>=0.18.0,!=0.11.0", + "huggingface_hub>=0.34.0", + "hf_transfer", + "diffusers", + "transformers>=4.51.3,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1,!=4.57.0,!=4.57.4,!=4.57.5,!=5.0.0,!=5.1.0,<=5.3.0", + "trl>=0.18.2,!=0.19.0,<=0.24.0", "typer", "pydantic", "pyyaml", @@ -64,14 +87,11 @@ studio = [ exclude = ["images*", "tests*", "kernels/moe*"] [project.optional-dependencies] -triton = [ - "triton>=3.0.0 ; ('linux' in sys_platform)", - "triton-windows ; (sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", -] - -huggingfacenotorch = [ +huggingface = [ + "unsloth_zoo>=2026.3.4", "wheel>=0.42.0", "packaging", + "torchvision", "numpy", "tqdm", "psutil", @@ -86,257 +106,132 @@ huggingfacenotorch = [ "diffusers", "transformers>=4.51.3,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1,!=4.57.0,!=4.57.4,!=4.57.5,!=5.0.0,!=5.1.0,<=5.3.0", "trl>=0.18.2,!=0.19.0,<=0.24.0", - "sentence-transformers", -] -huggingface = [ - "unsloth[huggingfacenotorch]", - "unsloth_zoo>=2026.3.4", - "torchvision", - "unsloth[triton]", -] -windows = [ - "unsloth[huggingface]", - "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0 ; (sys_platform == 'win32')", - "xformers>=0.0.22.post7 ; (sys_platform == 'win32')", + "typer", + "pydantic", + "pyyaml", + "nest-asyncio", ] base = [ "unsloth[huggingface]", ] cu118only = [ - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.22.post7%2Bcu118-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.22.post7%2Bcu118-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.22.post7%2Bcu118-cp311-cp311-manylinux2014_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", + "xformers==0.0.22.post7 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu121only = [ - "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.22.post7-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.22.post7-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.22.post7-cp311-cp311-manylinux2014_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", + "xformers==0.0.22.post7 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu118onlytorch211 = [ - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.23%2Bcu118-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.23%2Bcu118-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.23%2Bcu118-cp311-cp311-manylinux2014_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", + "xformers==0.0.23 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu121onlytorch211 = [ - "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.23-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.23-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.23-cp311-cp311-manylinux2014_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", + "xformers==0.0.23 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu118onlytorch212 = [ - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.23.post1%2Bcu118-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.23.post1%2Bcu118-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.23.post1%2Bcu118-cp311-cp311-manylinux2014_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", + "xformers==0.0.23.post1 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu121onlytorch212 = [ - "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.23.post1-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.23.post1-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.23.post1-cp311-cp311-manylinux2014_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", + "xformers==0.0.23.post1 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu118onlytorch220 = [ - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.24%2Bcu118-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.24%2Bcu118-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.24%2Bcu118-cp311-cp311-manylinux2014_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", + "xformers==0.0.24 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu121onlytorch220 = [ - "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.24-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.24-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.24-cp311-cp311-manylinux2014_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", + "xformers==0.0.24 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu118onlytorch230 = [ - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.27%2Bcu118-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.27%2Bcu118-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.27%2Bcu118-cp311-cp311-manylinux2014_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.27%2Bcu118-cp312-cp312-manylinux2014_x86_64.whl ; python_version=='3.12' and ('linux' in sys_platform)", + "xformers==0.0.27 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu121onlytorch230 = [ - "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.27-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.27-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.27-cp311-cp311-manylinux2014_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.27-cp312-cp312-manylinux2014_x86_64.whl ; python_version=='3.12' and ('linux' in sys_platform)", + "xformers==0.0.27 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu118onlytorch240 = [ - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.27.post2%2Bcu118-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.27.post2%2Bcu118-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.27.post2%2Bcu118-cp311-cp311-manylinux2014_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.27.post2%2Bcu118-cp312-cp312-manylinux2014_x86_64.whl ; python_version=='3.12' and ('linux' in sys_platform)", + "xformers==0.0.27.post2 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu121onlytorch240 = [ - "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.28.post1-cp39-cp39-manylinux_2_28_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.28.post1-cp310-cp310-manylinux_2_28_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.28.post1-cp311-cp311-manylinux_2_28_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.28.post1-cp312-cp312-manylinux_2_28_x86_64.whl ; python_version=='3.12' and ('linux' in sys_platform)", -] -cu124onlytorch240 = [ - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.28.post1-cp39-cp39-manylinux_2_28_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.28.post1-cp310-cp310-manylinux_2_28_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.28.post1-cp311-cp311-manylinux_2_28_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.28.post1-cp312-cp312-manylinux_2_28_x86_64.whl ; python_version=='3.12' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.28.post1-cp39-cp39-win_amd64.whl ; python_version=='3.9' and (sys_platform == 'win32')", - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.28.post1-cp310-cp310-win_amd64.whl ; python_version=='3.10' and (sys_platform == 'win32')", - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.28.post1-cp311-cp311-win_amd64.whl ; python_version=='3.11' and (sys_platform == 'win32')", - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.28.post1-cp312-cp312-win_amd64.whl ; python_version=='3.12' and (sys_platform == 'win32')", + "xformers==0.0.27.post2 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu118onlytorch250 = [ - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.28.post2-cp39-cp39-manylinux_2_28_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.28.post2-cp310-cp310-manylinux_2_28_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.28.post2-cp311-cp311-manylinux_2_28_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.28.post2-cp312-cp312-manylinux_2_28_x86_64.whl ; python_version=='3.12' and ('linux' in sys_platform)", + "xformers==0.0.28.post2 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu121onlytorch250 = [ - "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.28.post2-cp39-cp39-manylinux_2_28_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.28.post2-cp310-cp310-manylinux_2_28_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.28.post2-cp311-cp311-manylinux_2_28_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.28.post2-cp312-cp312-manylinux_2_28_x86_64.whl ; python_version=='3.12' and ('linux' in sys_platform)", + "xformers==0.0.28.post2 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu124onlytorch250 = [ - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.28.post2-cp39-cp39-manylinux_2_28_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.28.post2-cp310-cp310-manylinux_2_28_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.28.post2-cp311-cp311-manylinux_2_28_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.28.post2-cp312-cp312-manylinux_2_28_x86_64.whl ; python_version=='3.12' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.28.post2-cp39-cp39-win_amd64.whl ; python_version=='3.9' and (sys_platform == 'win32')", - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.28.post2-cp310-cp310-win_amd64.whl ; python_version=='3.10' and (sys_platform == 'win32')", - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.28.post2-cp311-cp311-win_amd64.whl ; python_version=='3.11' and (sys_platform == 'win32')", - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.28.post2-cp312-cp312-win_amd64.whl ; python_version=='3.12' and (sys_platform == 'win32')", + "xformers==0.0.28.post2 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu118onlytorch251 = [ - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.29.post1-cp39-cp39-manylinux_2_28_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.29.post1-cp310-cp310-manylinux_2_28_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.29.post1-cp311-cp311-manylinux_2_28_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.29.post1-cp312-cp312-manylinux_2_28_x86_64.whl ; python_version=='3.12' and ('linux' in sys_platform)", + "xformers==0.0.29.post1 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu121onlytorch251 = [ - "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.29.post1-cp39-cp39-manylinux_2_28_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.29.post1-cp310-cp310-manylinux_2_28_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.29.post1-cp311-cp311-manylinux_2_28_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.29.post1-cp312-cp312-manylinux_2_28_x86_64.whl ; python_version=='3.12' and ('linux' in sys_platform)", + "xformers==0.0.29.post1 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu124onlytorch251 = [ - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.29.post1-cp39-cp39-manylinux_2_28_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.29.post1-cp310-cp310-manylinux_2_28_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.29.post1-cp311-cp311-manylinux_2_28_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.29.post1-cp312-cp312-manylinux_2_28_x86_64.whl ; python_version=='3.12' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.29.post1-cp39-cp39-win_amd64.whl ; python_version=='3.9' and (sys_platform == 'win32')", - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.29.post1-cp310-cp310-win_amd64.whl ; python_version=='3.10' and (sys_platform == 'win32')", - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.29.post1-cp311-cp311-win_amd64.whl ; python_version=='3.11' and (sys_platform == 'win32')", - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.29.post1-cp312-cp312-win_amd64.whl ; python_version=='3.12' and (sys_platform == 'win32')", + "xformers==0.0.29.post1 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu118onlytorch260 = [ - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.29.post3-cp39-cp39-manylinux_2_28_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.29.post3-cp310-cp310-manylinux_2_28_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.29.post3-cp311-cp311-manylinux_2_28_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.29.post3-cp312-cp312-manylinux_2_28_x86_64.whl ; python_version=='3.12' and ('linux' in sys_platform)", + "xformers==0.0.29.post3 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu124onlytorch260 = [ - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.29.post3-cp39-cp39-manylinux_2_28_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.29.post3-cp310-cp310-manylinux_2_28_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.29.post3-cp311-cp311-manylinux_2_28_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.29.post3-cp312-cp312-manylinux_2_28_x86_64.whl ; python_version=='3.12' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.29.post3-cp39-cp39-win_amd64.whl ; python_version=='3.9' and (sys_platform == 'win32')", - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.29.post3-cp310-cp310-win_amd64.whl ; python_version=='3.10' and (sys_platform == 'win32')", - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.29.post3-cp311-cp311-win_amd64.whl ; python_version=='3.11' and (sys_platform == 'win32')", - "xformers @ https://download.pytorch.org/whl/cu124/xformers-0.0.29.post3-cp312-cp312-win_amd64.whl ; python_version=='3.12' and (sys_platform == 'win32')", + "xformers==0.0.29.post3 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu126onlytorch260 = [ - "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.29.post3-cp39-cp39-manylinux_2_28_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.29.post3-cp310-cp310-manylinux_2_28_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.29.post3-cp311-cp311-manylinux_2_28_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.29.post3-cp312-cp312-manylinux_2_28_x86_64.whl ; python_version=='3.12' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.29.post3-cp39-cp39-win_amd64.whl ; python_version=='3.9' and (sys_platform == 'win32')", - "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.29.post3-cp310-cp310-win_amd64.whl ; python_version=='3.10' and (sys_platform == 'win32')", - "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.29.post3-cp311-cp311-win_amd64.whl ; python_version=='3.11' and (sys_platform == 'win32')", - "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.29.post3-cp312-cp312-win_amd64.whl ; python_version=='3.12' and (sys_platform == 'win32')", + "xformers==0.0.29.post3 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu118onlytorch270 = [ - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.30-cp39-cp39-manylinux_2_28_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.30-cp310-cp310-manylinux_2_28_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.30-cp311-cp311-manylinux_2_28_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.30-cp312-cp312-manylinux_2_28_x86_64.whl ; python_version=='3.12' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.30-cp39-cp39-win_amd64.whl ; python_version=='3.9' and (sys_platform == 'win32')", - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.30-cp310-cp310-win_amd64.whl ; python_version=='3.10' and (sys_platform == 'win32')", - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.30-cp311-cp311-win_amd64.whl ; python_version=='3.11' and (sys_platform == 'win32')", - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.30-cp312-cp312-win_amd64.whl ; python_version=='3.12' and (sys_platform == 'win32')", + "xformers==0.0.30 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu126onlytorch270 = [ - "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.30-cp39-cp39-manylinux_2_28_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.30-cp310-cp310-manylinux_2_28_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.30-cp311-cp311-manylinux_2_28_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.30-cp312-cp312-manylinux_2_28_x86_64.whl ; python_version=='3.12' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.30-cp39-cp39-win_amd64.whl ; python_version=='3.9' and (sys_platform == 'win32')", - "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.30-cp310-cp310-win_amd64.whl ; python_version=='3.10' and (sys_platform == 'win32')", - "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.30-cp311-cp311-win_amd64.whl ; python_version=='3.11' and (sys_platform == 'win32')", - "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.30-cp312-cp312-win_amd64.whl ; python_version=='3.12' and (sys_platform == 'win32')", + "xformers==0.0.30 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu128onlytorch270 = [ - "xformers @ https://download.pytorch.org/whl/cu128/xformers-0.0.30-cp39-cp39-manylinux_2_28_x86_64.whl ; python_version=='3.9' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu128/xformers-0.0.30-cp310-cp310-manylinux_2_28_x86_64.whl ; python_version=='3.10' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu128/xformers-0.0.30-cp311-cp311-manylinux_2_28_x86_64.whl ; python_version=='3.11' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu128/xformers-0.0.30-cp312-cp312-manylinux_2_28_x86_64.whl ; python_version=='3.12' and ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu128/xformers-0.0.30-cp39-cp39-win_amd64.whl ; python_version=='3.9' and (sys_platform == 'win32')", - "xformers @ https://download.pytorch.org/whl/cu128/xformers-0.0.30-cp310-cp310-win_amd64.whl ; python_version=='3.10' and (sys_platform == 'win32')", - "xformers @ https://download.pytorch.org/whl/cu128/xformers-0.0.30-cp311-cp311-win_amd64.whl ; python_version=='3.11' and (sys_platform == 'win32')", - "xformers @ https://download.pytorch.org/whl/cu128/xformers-0.0.30-cp312-cp312-win_amd64.whl ; python_version=='3.12' and (sys_platform == 'win32')", + "xformers==0.0.30 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu118onlytorch271 = [ - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.31.post1-cp39-abi3-manylinux_2_28_x86_64.whl ; ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.31.post1-cp39-abi3-win_amd64.whl ; (sys_platform == 'win32')", + "xformers==0.0.31.post1 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu126onlytorch271 = [ - "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.31.post1-cp39-abi3-manylinux_2_28_x86_64.whl ; ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.31.post1-cp39-abi3-win_amd64.whl ; (sys_platform == 'win32')", + "xformers==0.0.31.post1 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu128onlytorch271 = [ - "xformers @ https://download.pytorch.org/whl/cu128/xformers-0.0.31.post1-cp39-abi3-manylinux_2_28_x86_64.whl ; ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu128/xformers-0.0.31.post1-cp39-abi3-win_amd64.whl ; (sys_platform == 'win32')", + "xformers==0.0.31.post1 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu118onlytorch280 = [ - "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.32.post2-cp39-abi3-manylinux_2_28_x86_64.whl ; ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.32.post2-cp39-abi3-win_amd64.whl ; (sys_platform == 'win32')", + "xformers==0.0.32.post2 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu126onlytorch280 = [ - "xformers @ https://download.pytorch.org/whl/cu128/xformers-0.0.32.post2-cp39-abi3-manylinux_2_28_x86_64.whl ; ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu128/xformers-0.0.32.post2-cp39-abi3-win_amd64.whl ; (sys_platform == 'win32')", + "xformers==0.0.32.post2 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu128onlytorch280 = [ - "xformers @ https://download.pytorch.org/whl/cu129/xformers-0.0.32.post2-cp39-abi3-manylinux_2_28_x86_64.whl ; ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu129/xformers-0.0.32.post2-cp39-abi3-win_amd64.whl ; (sys_platform == 'win32')", + "xformers==0.0.32.post2 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu130onlytorch280 = [ ] cu126onlytorch290 = [ - "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.33.post1-cp39-abi3-manylinux_2_28_x86_64.whl ; ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.33.post1-cp39-abi3-win_amd64.whl ; (sys_platform == 'win32')", + "xformers==0.0.33.post1 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu128onlytorch290 = [ - "xformers @ https://download.pytorch.org/whl/cu128/xformers-0.0.33.post1-cp39-abi3-manylinux_2_28_x86_64.whl ; ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu128/xformers-0.0.33.post1-cp39-abi3-win_amd64.whl ; (sys_platform == 'win32')", + "xformers==0.0.33.post1 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu130onlytorch290 = [ - "xformers @ https://download.pytorch.org/whl/cu130/xformers-0.0.33.post1-cp39-abi3-manylinux_2_28_x86_64.whl ; ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu130/xformers-0.0.33.post1-cp39-abi3-win_amd64.whl ; (sys_platform == 'win32')", + "xformers==0.0.33.post1 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu126onlytorch291 = [ - "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.33.post2-cp39-abi3-manylinux_2_28_x86_64.whl ; ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.33.post2-cp39-abi3-win_amd64.whl ; (sys_platform == 'win32')", + "xformers==0.0.33.post2 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu128onlytorch291 = [ - "xformers @ https://download.pytorch.org/whl/cu128/xformers-0.0.33.post2-cp39-abi3-manylinux_2_28_x86_64.whl ; ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu128/xformers-0.0.33.post2-cp39-abi3-win_amd64.whl ; (sys_platform == 'win32')", + "xformers==0.0.33.post2 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu130onlytorch291 = [ - "xformers @ https://download.pytorch.org/whl/cu130/xformers-0.0.33.post2-cp39-abi3-manylinux_2_28_x86_64.whl ; ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu130/xformers-0.0.33.post2-cp39-abi3-win_amd64.whl ; (sys_platform == 'win32')", + "xformers==0.0.33.post2 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu126onlytorch2100 = [ - "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.34-cp39-abi3-manylinux_2_28_x86_64.whl ; ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.34-cp39-abi3-win_amd64.whl ; (sys_platform == 'win32')", + "xformers==0.0.34 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu128onlytorch2100 = [ - "xformers @ https://download.pytorch.org/whl/cu128/xformers-0.0.34-cp39-abi3-manylinux_2_28_x86_64.whl ; ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu128/xformers-0.0.34-cp39-abi3-win_amd64.whl ; (sys_platform == 'win32')", + "xformers==0.0.34 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu130onlytorch2100 = [ - "xformers @ https://download.pytorch.org/whl/cu130/xformers-0.0.34-cp39-abi3-manylinux_2_28_x86_64.whl ; ('linux' in sys_platform)", - "xformers @ https://download.pytorch.org/whl/cu130/xformers-0.0.34-cp39-abi3-win_amd64.whl ; (sys_platform == 'win32')", + "xformers==0.0.34 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] cu118 = [ "unsloth[huggingface]", @@ -350,22 +245,22 @@ cu121 = [ ] cu118-torch211 = [ "unsloth[huggingface]", - "bitsandbytes==0.45.5", + "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", "unsloth[cu118onlytorch211]", ] cu121-torch211 = [ "unsloth[huggingface]", - "bitsandbytes==0.45.5", + "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", "unsloth[cu121onlytorch211]", ] cu118-torch212 = [ "unsloth[huggingface]", - "bitsandbytes==0.45.5", + "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", "unsloth[cu118onlytorch212]", ] cu121-torch212 = [ "unsloth[huggingface]", - "bitsandbytes==0.45.5", + "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", "unsloth[cu121onlytorch212]", ] cu118-torch220 = [ @@ -435,17 +330,17 @@ cu124-torch251 = [ ] cu118-torch260 = [ "unsloth[huggingface]", - "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", + "bitsandbytes>=0.45.1", "unsloth[cu118onlytorch260]", ] cu124-torch260 = [ "unsloth[huggingface]", - "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", + "bitsandbytes>=0.45.1", "unsloth[cu124onlytorch260]", ] cu126-torch260 = [ "unsloth[huggingface]", - "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", + "bitsandbytes>=0.45.1", "unsloth[cu126onlytorch260]", ] cu118-torch270 = [ @@ -555,16 +450,19 @@ conda = [ ] colab-torch211 = [ "unsloth[huggingface]", - "bitsandbytes==0.45.5", + "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", "unsloth[cu121onlytorch211]", ] +flashattention = [ + "packaging ; ('linux' in sys_platform)", + "ninja ; ('linux' in sys_platform)", + "flash-attn>=2.6.3 ; ('linux' in sys_platform)", +] colab-ampere-torch211 = [ "unsloth[huggingface]", - "bitsandbytes==0.45.5", + "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", "unsloth[cu121onlytorch211]", - "packaging", - "ninja", - "flash-attn>=2.6.3 ; ('linux' in sys_platform)", + "unsloth[flashattention]", ] colab-torch220 = [ "unsloth[huggingface]", @@ -575,9 +473,7 @@ colab-ampere-torch220 = [ "unsloth[huggingface]", "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", "unsloth[cu121onlytorch220]", - "packaging", - "ninja", - "flash-attn>=2.6.3 ; ('linux' in sys_platform)", + "unsloth[flashattention]", ] colab-new = [ "unsloth_zoo>=2026.3.4", @@ -595,24 +491,22 @@ colab-new = [ "hf_transfer", "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", "unsloth[triton]", - "sentence-transformers", + "typer", + "pydantic", + "pyyaml", + "nest-asyncio", ] colab-no-deps = [ "accelerate>=0.34.1", "trl>=0.18.2,!=0.19.0,<=0.24.0", "peft>=0.18.0", - "xformers ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", + "xformers<0.0.27", "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", "protobuf", ] colab = [ "unsloth[cu121]", ] -flashattention = [ - "packaging ; ('linux' in sys_platform)", - "ninja ; ('linux' in sys_platform)", - "flash-attn>=2.6.3 ; ('linux' in sys_platform)", -] colab-ampere = [ "unsloth[colab-ampere-torch220]", "unsloth[flashattention]", @@ -631,13 +525,13 @@ cu121-ampere = [ ] cu118-ampere-torch211 = [ "unsloth[huggingface]", - "bitsandbytes==0.45.5", + "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", "unsloth[cu118onlytorch211]", "unsloth[flashattention]", ] cu121-ampere-torch211 = [ "unsloth[huggingface]", - "bitsandbytes==0.45.5", + "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", "unsloth[cu121onlytorch211]", "unsloth[flashattention]", ] @@ -721,19 +615,19 @@ cu124-ampere-torch251 = [ ] cu118-ampere-torch260 = [ "unsloth[huggingface]", - "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", + "bitsandbytes>=0.45.1", "unsloth[cu118onlytorch260]", "unsloth[flashattention]", ] cu124-ampere-torch260 = [ "unsloth[huggingface]", - "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", + "bitsandbytes>=0.45.1", "unsloth[cu124onlytorch260]", "unsloth[flashattention]", ] cu126-ampere-torch260 = [ "unsloth[huggingface]", - "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", + "bitsandbytes>=0.45.1", "unsloth[cu126onlytorch260]", "unsloth[flashattention]", ] @@ -795,7 +689,6 @@ cu130-ampere-torch280 = [ "unsloth[huggingface]", "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", "unsloth[cu130onlytorch280]", - "unsloth[flashattention]", ] cu126-ampere-torch290 = [ "unsloth[huggingface]", @@ -842,347 +735,8 @@ cu130-ampere-torch2100 = [ "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", "unsloth[cu130onlytorch2100]", ] -flashattentiontorch260abiFALSEcu12x = [ - "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.6cxx11abiFALSE-cp39-cp39-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.9'", - "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.6cxx11abiFALSE-cp310-cp310-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.10'", - "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.6cxx11abiFALSE-cp311-cp311-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.11'", - "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.6cxx11abiFALSE-cp312-cp312-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.12'", - "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.6cxx11abiFALSE-cp313-cp313-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.13'", -] -flashattentiontorch260abiTRUEcu12x = [ - "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.6cxx11abiTRUE-cp39-cp39-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.9'", - "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.6cxx11abiTRUE-cp310-cp310-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.10'", - "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.6cxx11abiTRUE-cp311-cp311-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.11'", - "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.6cxx11abiTRUE-cp312-cp312-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.12'", - "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.6cxx11abiTRUE-cp313-cp313-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.13'", -] -flashattentiontorch250abiFALSEcu12x = [ - "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.5cxx11abiFALSE-cp39-cp39-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.9'", - "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.5cxx11abiFALSE-cp310-cp310-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.10'", - "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.5cxx11abiFALSE-cp311-cp311-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.11'", - "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.5cxx11abiFALSE-cp312-cp312-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.12'", - "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.5cxx11abiFALSE-cp313-cp313-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.13'", -] -flashattentiontorch250abiTRUEcu12x = [ - "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.5cxx11abiTRUE-cp39-cp39-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.9'", - "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.5cxx11abiTRUE-cp310-cp310-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.10'", - "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.5cxx11abiTRUE-cp311-cp311-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.11'", - "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.5cxx11abiTRUE-cp312-cp312-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.12'", - "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.5cxx11abiTRUE-cp313-cp313-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.13'", -] -flashattentiontorch240abiFALSEcu12x = [ - "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.4cxx11abiFALSE-cp39-cp39-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.9'", - "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.4cxx11abiFALSE-cp310-cp310-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.10'", - "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.4cxx11abiFALSE-cp311-cp311-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.11'", - "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.4cxx11abiFALSE-cp312-cp312-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.12'", - "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.4cxx11abiFALSE-cp313-cp313-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.13'", -] -flashattentiontorch240abiTRUEcu12x = [ - "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.4cxx11abiTRUE-cp39-cp39-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.9'", - "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.4cxx11abiTRUE-cp310-cp310-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.10'", - "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.4cxx11abiTRUE-cp311-cp311-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.11'", - "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.4cxx11abiTRUE-cp312-cp312-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.12'", - "flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.4cxx11abiTRUE-cp313-cp313-linux_x86_64.whl ; ('linux' in sys_platform) and python_version == '3.13'", -] -intelgputorch260 = [ - "unsloth_zoo[intelgpu]", - "unsloth[huggingfacenotorch]", - - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.2.0-cp39-cp39-linux_x86_64.whl#sha256=147607f190a7d7aa24ba454def5977fbbfec792fdae18e4ed278cfec29b69271 ; ('linux' in sys_platform) and python_version == '3.9' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.2.0-cp310-cp310-linux_x86_64.whl#sha256=23aa423fa1542afc34f67eb3ba8ef20060f6d1b3a4697eaeab22b11c92b30f2b ; ('linux' in sys_platform) and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.2.0-cp311-cp311-linux_x86_64.whl#sha256=bcfa995229bbfd9ffd8d6c8d9f6428d393e876fa6e23ee3c20e3c0d73ca75ca5 ; ('linux' in sys_platform) and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.2.0-cp312-cp312-linux_x86_64.whl#sha256=bd340903d03470708df3442438acb8b7e08087ab9e61fbe349b2872bf9257ab0 ; ('linux' in sys_platform) and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.2.0-cp313-cp313-linux_x86_64.whl#sha256=814dccc8a07159e6eca74bed70091bc8fea2d9dd87b0d91845f9f38cde62f01c ; ('linux' in sys_platform) and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - - "bitsandbytes @ https://github.com/bitsandbytes-foundation/bitsandbytes/releases/download/continuous-release_main/bitsandbytes-1.33.7.preview-py3-none-manylinux_2_24_x86_64.whl ; ('linux' in sys_platform) and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "bitsandbytes @ https://github.com/bitsandbytes-foundation/bitsandbytes/releases/download/continuous-release_main/bitsandbytes-1.33.7.preview-py3-none-win_amd64.whl ; (sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - - "torch @ https://download.pytorch.org/whl/xpu/torch-2.6.0%2Bxpu-cp39-cp39-linux_x86_64.whl#sha256=6a8adf6dc4c089406e8b3a7e58ab57a463bddf9b07130d2576e76eced43e92af ; ('linux' in sys_platform) and python_version == '3.9' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.6.0%2Bxpu-cp310-cp310-linux_x86_64.whl#sha256=ff4561cbf07c83bbccaa0f6e9bb0e6dcf721bacd53c9c43c4eb0e7331b4792f9 ; ('linux' in sys_platform) and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.6.0%2Bxpu-cp311-cp311-linux_x86_64.whl#sha256=12005f66b810ddd3ab93f86c4522bcfdd412cbd27fc9d189b661ff7509bc5e8a ; ('linux' in sys_platform) and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.6.0%2Bxpu-cp312-cp312-linux_x86_64.whl#sha256=c4c5c67625cdacf35765c2b94e61fe166e3c3f4a14521b1212a59ad1b3eb0f2e ; ('linux' in sys_platform) and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.6.0%2Bxpu-cp313-cp313-linux_x86_64.whl#sha256=e6864f7a60a5ecc43d5d38f59a16e5dd132384f73dfd3a697f74944026038f7b ; ('linux' in sys_platform) and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", -] -intel-gpu-torch260 = [ - "unsloth[intelgputorch260]" -] -intelgputorch270 = [ - "unsloth_zoo[intelgpu]", - "unsloth[huggingfacenotorch]", - - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.3.0-cp39-cp39-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=749a7098492c6a27b356c97149a4a62973b953eae60bc1b6259260974f344913 ; ('linux' in sys_platform) and python_version == '3.9' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.3.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=44362e80abd752471a08341093321955b066daa2cfb4810e73b8e3b240850f93 ; ('linux' in sys_platform) and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.3.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=faa6b8c945a837a080f641bc8ccc77a98fa66980dcd7e62e715fd853737343fd ; ('linux' in sys_platform) and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.3.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=40f6fb65b345dc9a61813abe7ac9a585f2c9808f414d140cc2a5f11f53ee063c ; ('linux' in sys_platform) and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.3.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=b22b4c02ec71b4bfc862ae3cdfd2871dc0b05d2b1802f5db2196e0f897d581e9 ; ('linux' in sys_platform) and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.3.0-cp39-cp39-win_amd64.whl#sha256=d4b738d7fa5100c1bd766f91614962828a4810eb57b4df92cd5214a83505a752 ; sys_platform == 'win32' and python_version == '3.9' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.3.0-cp310-cp310-win_amd64.whl#sha256=143fe8a64d807bcdb7d81bbc062816add325570aa160448454ab6ded4a0a17a1 ; sys_platform == 'win32' and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.3.0-cp311-cp311-win_amd64.whl#sha256=a8025459ff325d6e3532eb5cf72519db1b178155e7d60aff6c56beb5968fc758 ; sys_platform == 'win32' and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.3.0-cp312-cp312-win_amd64.whl#sha256=0dd07e6d5b872e42e48f5ee140e609d4554ca3cc509d5bf509ac232267cf358e ; sys_platform == 'win32' and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.3.0-cp313-cp313-win_amd64.whl#sha256=a936a18182d8e065a9933afc9a3ebbffadd38604969f87c493831214539fc027 ; sys_platform == 'win32' and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - - "bitsandbytes @ https://github.com/bitsandbytes-foundation/bitsandbytes/releases/download/continuous-release_main/bitsandbytes-1.33.7.preview-py3-none-manylinux_2_24_x86_64.whl ; ('linux' in sys_platform) and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "bitsandbytes @ https://github.com/bitsandbytes-foundation/bitsandbytes/releases/download/continuous-release_main/bitsandbytes-1.33.7.preview-py3-none-win_amd64.whl ; (sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - - "torch @ https://download.pytorch.org/whl/xpu/torch-2.7.0%2Bxpu-cp39-cp39-linux_x86_64.whl#sha256=f8ee75e50fcbb37ed5b498299ca2264da99ab278a93fae2358e921e4a6e28273 ; ('linux' in sys_platform) and python_version == '3.9' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.7.0%2Bxpu-cp310-cp310-linux_x86_64.whl#sha256=d6fdc342961d98fdcd9d03dfd491a3208bb5f7fbb435841f8f72ce9fdcd2d026 ; ('linux' in sys_platform) and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.7.0%2Bxpu-cp311-cp311-linux_x86_64.whl#sha256=74d07f9357df5cf2bf223ad3c84de16346bfaa0504f988fdd5590d3e177e5e86 ; ('linux' in sys_platform) and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.7.0%2Bxpu-cp312-cp312-linux_x86_64.whl#sha256=c806d44aa2ca5d225629f6fbc6c994d5deaac2d2cde449195bc8e3522ddd219a ; ('linux' in sys_platform) and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.7.0%2Bxpu-cp313-cp313-linux_x86_64.whl#sha256=25d8277b7f01d42e2e014ccbab57a2692b6ec4eff8dcf894eda1b297407cf97a ; ('linux' in sys_platform) and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.7.0%2Bxpu-cp39-cp39-win_amd64.whl#sha256=046e85125266ae69c1a0d083e6c092f947ab4b6b41532c16bafe40dbced845df ; sys_platform == 'win32' and python_version == '3.9' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.7.0%2Bxpu-cp310-cp310-win_amd64.whl#sha256=9ebaeffb82b0b3e39b6030927d3ebe0eb62a0e9045a3b2d7b0a9e7b15222c0db ; sys_platform == 'win32' and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.7.0%2Bxpu-cp311-cp311-win_amd64.whl#sha256=356ba66cee127e7e2c942880bd50e03768306a4ea08d358a0f29c6eebfc4bc81 ; sys_platform == 'win32' and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.7.0%2Bxpu-cp312-cp312-win_amd64.whl#sha256=94739e665d9b4d5cd7af5f517cb6103f6f9fb421c095184609653a24524040f5 ; sys_platform == 'win32' and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.7.0%2Bxpu-cp313-cp313-win_amd64.whl#sha256=31df3cb674918e89bc8c532baa331dc84f4430e1f9c0ec379232db44cba78355 ; sys_platform == 'win32' and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", -] -intel-gpu-torch270 = [ - "unsloth[intelgputorch270]" -] -intelgputorch280 = [ - "unsloth_zoo[intelgpu]", - "unsloth[huggingfacenotorch]", - - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.4.0-cp39-cp39-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=ac4d8e33986b1c3c5e48151640539272b2187e83016985853111b46fb82c3c94 ; 'linux' in sys_platform and python_version == '3.9' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.4.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=999fef4c1f711092b9d3086525920545df490de476ecebe899ffc777019ae17f ; 'linux' in sys_platform and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.4.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=57b09c8c492985ff6a27cd3a22b08e8f7b96b407bd8030967b6efbb9f63b80cf ; 'linux' in sys_platform and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.4.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=df4bb3282bac9a3b90231700077110d8680b338416de03c2b7c6133c9b602649 ; 'linux' in sys_platform and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.4.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=60da63c99ca827bdcb0df28e0298bf7d066dc607454c6d6176783cb4e79d838b ; 'linux' in sys_platform and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.4.0-cp39-cp39-win_amd64.whl#sha256=64aea8de349f3e2e0ebf4c24b011a8122531fdffda5776edaef45829cc241cf8 ; sys_platform == 'win32' and python_version == '3.9' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.4.0-cp310-cp310-win_amd64.whl#sha256=ae573d255b257fdbed319a3440dc9d0a721e31160ab7f6eba1b2226e6a409a1d ; sys_platform == 'win32' and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.4.0-cp311-cp311-win_amd64.whl#sha256=8e0ea4558e5776d8ddab0264310be9b26aee5641bcac0da023537556d4317b86 ; sys_platform == 'win32' and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.4.0-cp312-cp312-win_amd64.whl#sha256=4090dde07a4fffc34aaf855701a9db28e9fccb57b368ade520f1a0f8e811c878 ; sys_platform == 'win32' and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.4.0-cp313-cp313-win_amd64.whl#sha256=a33d0888f3c8df028a2d028842715837d0049524d6c06b9bb11869890a13601a ; sys_platform == 'win32' and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - - "torch @ https://download.pytorch.org/whl/xpu/torch-2.8.0%2Bxpu-cp39-cp39-linux_x86_64.whl ; 'linux' in sys_platform and python_version == '3.9' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.8.0%2Bxpu-cp310-cp310-linux_x86_64.whl ; 'linux' in sys_platform and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.8.0%2Bxpu-cp311-cp311-linux_x86_64.whl ; 'linux' in sys_platform and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.8.0%2Bxpu-cp312-cp312-linux_x86_64.whl ; 'linux' in sys_platform and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.8.0%2Bxpu-cp313-cp313-linux_x86_64.whl ; 'linux' in sys_platform and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.8.0%2Bxpu-cp39-cp39-win_amd64.whl#sha256=f2f401276892428e4875cf1d8717c5cbab704b16fc594ccf23795e7b16549a99 ; sys_platform == 'win32' and python_version == '3.9' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.8.0%2Bxpu-cp310-cp310-win_amd64.whl#sha256=125c60cd59d51b39581a7e9afcd4679bc3a6b8c1f9440b1bb502a23fdd60571e ; sys_platform == 'win32' and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.8.0%2Bxpu-cp311-cp311-win_amd64.whl#sha256=47f1a57258cd460e80b38b2ed6744e31587ab77a96b4215bf59546cb4bab5cc0 ; sys_platform == 'win32' and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.8.0%2Bxpu-cp312-cp312-win_amd64.whl#sha256=0937d8943c145a83d9bafc6f80ef28971167817f9eda26066d33f72caf8a6646 ; sys_platform == 'win32' and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.8.0%2Bxpu-cp313-cp313-win_amd64.whl#sha256=e034aab1d71760dc80a731531be43673ffe15e99033b82d24e40d2e6d41bd8bf ; sys_platform == 'win32' and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - - "bitsandbytes @ https://github.com/bitsandbytes-foundation/bitsandbytes/releases/download/continuous-release_main/bitsandbytes-1.33.7.preview-py3-none-manylinux_2_24_x86_64.whl ; ('linux' in sys_platform) and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "bitsandbytes @ https://github.com/bitsandbytes-foundation/bitsandbytes/releases/download/continuous-release_main/bitsandbytes-1.33.7.preview-py3-none-win_amd64.whl ; (sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.23.0%2Bxpu-cp39-cp39-manylinux_2_28_x86_64.whl#sha256=6e981c192045fc249c008441179ff237bb00174d818b875b0475730b63f0eaca ; 'linux' in sys_platform and python_version == '3.9' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.23.0%2Bxpu-cp310-cp310-manylinux_2_28_x86_64.whl#sha256=e5ba4805969277175ebfd59cc717093528cc6e3ada89ac2725fc7a3c1fee6169 ; 'linux' in sys_platform and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.23.0%2Bxpu-cp311-cp311-manylinux_2_28_x86_64.whl#sha256=74c39c144104416bc4c5ad8c26ab0c169dc5cc6be58059e01bc3665dd0ef676f ; 'linux' in sys_platform and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.23.0%2Bxpu-cp312-cp312-manylinux_2_28_x86_64.whl#sha256=0acec355b80c3899841184084f365df336c508602812e34a44007b8b60d53af4 ; 'linux' in sys_platform and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.23.0%2Bxpu-cp313-cp313-manylinux_2_28_x86_64.whl#sha256=e2109ae773dad27b98ca17681044b4f876563c37f2382b75de3a371399edcff8 ; 'linux' in sys_platform and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.23.0%2Bxpu-cp39-cp39-win_amd64.whl#sha256=5f7904e7048d414379bc8c1167260f1e84204f105db2d0a2f9c89e87ce1cf205 ; sys_platform == 'win32' and python_version == '3.9' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.23.0%2Bxpu-cp310-cp310-win_amd64.whl#sha256=005fca5e658ca8e37adb63c1a021c84f5e56dfa6cf0d601d89cfe40b9473f79f ; sys_platform == 'win32' and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.23.0%2Bxpu-cp311-cp311-win_amd64.whl#sha256=c6d030f5361461550c0ff1339b5bca8585fc1e84fda2e64b6184e65a581e4f98 ; sys_platform == 'win32' and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.23.0%2Bxpu-cp312-cp312-win_amd64.whl#sha256=91aafd61864cdce27461cbec13ddbf28c1bc6494265a1e4b80131c64a3b7d18f ; sys_platform == 'win32' and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.23.0%2Bxpu-cp313-cp313-win_amd64.whl#sha256=71dc4a6421742ed1e7f585b04a100ad53615c341fbccfbc255aefb38ea9091da ; sys_platform == 'win32' and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", -] -intel-gpu-torch280 = [ - "unsloth[intelgputorch280]" -] -intelgputorch290 = [ - "unsloth_zoo[intelgpu]", - "unsloth[huggingfacenotorch]", - - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.5.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=c169a1de14c19673b17c751290d467fa282fc90fa5da4314b2e5cdab1f553146 ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.5.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=013d9dd5d6479bd22983161f462e61c8dbe1d82e6730624a7a8d5945507eaa61 ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.5.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=afc8cabfbf7ed51fd278d1e0f88d6afc157b0201bad4b99d681e4d542f9e66d4 ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.5.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=0d24c1716088f2764d0d24c64227732195b6a42706c3c5fc89eeb4904bfa0818 ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.5.0-cp310-cp310-win_amd64.whl#sha256=c83ab007311d9cfb6e809ee5a4587d99a9eef4be720b90da4f1aaa68b45139a0 ; sys_platform == 'win32' and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.5.0-cp311-cp311-win_amd64.whl#sha256=debf75348da8e8c7166b4d4a9b91d1508bb8d6581e339f79f7604b2e6746bacd ; sys_platform == 'win32' and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.5.0-cp312-cp312-win_amd64.whl#sha256=97337a47425f1963a723475bd61037460e84ba01db4f87a1d662c3718ff6c47e ; sys_platform == 'win32' and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.5.0-cp313-cp313-win_amd64.whl#sha256=2caf8138695f6abb023ecd02031a2611ba1bf8fff2f19802567cb2fadefe9e87 ; sys_platform == 'win32' and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - - "torch @ https://download.pytorch.org/whl/xpu/torch-2.9.0%2Bxpu-cp310-cp310-linux_x86_64.whl#sha256=5afbe860ce991825a36b75706a523601087e414b77598ef0d9d3d565741c277d ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.9.0%2Bxpu-cp311-cp311-linux_x86_64.whl#sha256=607fe419c32d6e8e0556f745742e7cff1d0babce51f54be890e0c1422359c442 ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.9.0%2Bxpu-cp312-cp312-linux_x86_64.whl#sha256=376bae584d89980b8e59934d248c38d5fa3b7d4687a4df1a19f4bc1d23dcc8c1 ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.9.0%2Bxpu-cp313-cp313-linux_x86_64.whl#sha256=98d6a06dd7fb185874367b18bd609f05f16fdce4142a5980ca94461949965cd2 ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.9.0%2Bxpu-cp310-cp310-win_amd64.whl#sha256=47cc68f631f65bd9c84924d052cd04dec7531023caa85e80345e9c94611c887d ; sys_platform == 'win32' and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.9.0%2Bxpu-cp311-cp311-win_amd64.whl#sha256=d56c44ab4818aba57e5c7b628f422d014e0d507427170a771c5be85e308b0bc6 ; sys_platform == 'win32' and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.9.0%2Bxpu-cp312-cp312-win_amd64.whl#sha256=18cad93aaff76a01ce73aef6935ece7cfc03344b905592ec731446c44d44592b ; sys_platform == 'win32' and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.9.0%2Bxpu-cp313-cp313-win_amd64.whl#sha256=579929cdc10a76800ead41289cac191ea36d1b16f5f501d3fc25607d4375cd83 ; sys_platform == 'win32' and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - - "bitsandbytes @ https://github.com/bitsandbytes-foundation/bitsandbytes/releases/download/continuous-release_main/bitsandbytes-1.33.7.preview-py3-none-manylinux_2_24_x86_64.whl ; ('linux' in sys_platform) and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "bitsandbytes @ https://github.com/bitsandbytes-foundation/bitsandbytes/releases/download/continuous-release_main/bitsandbytes-1.33.7.preview-py3-none-win_amd64.whl ; (sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.24.0%2Bxpu-cp310-cp310-manylinux_2_28_x86_64.whl#sha256=cbfae2b79b7549fd368c2462fc8e94f8f26cc450782ee72138e908077c09a519 ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.24.0%2Bxpu-cp311-cp311-manylinux_2_28_x86_64.whl#sha256=044fa36ef4b6b43edcd490b75c853fa4b3eb033c2bded29f8fbcf27734713c67 ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.24.0%2Bxpu-cp312-cp312-manylinux_2_28_x86_64.whl#sha256=4b91e4bec1d740a6211f02578a79888550b73f3a4e1383035f8f6d72f587212c ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.24.0%2Bxpu-cp313-cp313-manylinux_2_28_x86_64.whl#sha256=88239e73ca37254bec84f29cd5887e10ff712de7edbbda3fbb3609cd6190d99e ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.24.0%2Bxpu-cp310-cp310-win_amd64.whl#sha256=19c7da8ca767d593e13a88a12bb08d06e34a673f6f26c2f9c191d60e81c02953 ; sys_platform == 'win32' and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.24.0%2Bxpu-cp311-cp311-win_amd64.whl#sha256=9bb0d1421c544ac8e2eca5b47daacaf54706dc9139c003aa5e77ee5f355c5931 ; sys_platform == 'win32' and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.24.0%2Bxpu-cp312-cp312-win_amd64.whl#sha256=6a5194bc736089606342d48a3f6822829b167617e9495d91d753dd1bd46fda18 ; sys_platform == 'win32' and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.24.0%2Bxpu-cp313-cp313-win_amd64.whl#sha256=da47a3ce2bb7f0301a31124668b5908f9b9e92d6241443de15a310ef9632fd83 ; sys_platform == 'win32' and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", -] -intel-gpu-torch290 = [ - "unsloth[intelgputorch290]" -] -intelgputorch210 = [ - "unsloth_zoo[intelgpu]", - "unsloth[huggingfacenotorch]", - - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.5.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=c169a1de14c19673b17c751290d467fa282fc90fa5da4314b2e5cdab1f553146 ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.5.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=013d9dd5d6479bd22983161f462e61c8dbe1d82e6730624a7a8d5945507eaa61 ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.5.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=afc8cabfbf7ed51fd278d1e0f88d6afc157b0201bad4b99d681e4d542f9e66d4 ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.5.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=0d24c1716088f2764d0d24c64227732195b6a42706c3c5fc89eeb4904bfa0818 ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.5.0-cp310-cp310-win_amd64.whl#sha256=c83ab007311d9cfb6e809ee5a4587d99a9eef4be720b90da4f1aaa68b45139a0 ; sys_platform == 'win32' and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.5.0-cp311-cp311-win_amd64.whl#sha256=debf75348da8e8c7166b4d4a9b91d1508bb8d6581e339f79f7604b2e6746bacd ; sys_platform == 'win32' and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.5.0-cp312-cp312-win_amd64.whl#sha256=97337a47425f1963a723475bd61037460e84ba01db4f87a1d662c3718ff6c47e ; sys_platform == 'win32' and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.5.0-cp313-cp313-win_amd64.whl#sha256=2caf8138695f6abb023ecd02031a2611ba1bf8fff2f19802567cb2fadefe9e87 ; sys_platform == 'win32' and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - - "torch @ https://download.pytorch.org/whl/xpu/torch-2.10.0%2Bxpu-cp310-cp310-linux_x86_64.whl#sha256=abb1d1ec1ac672bac0ff35420c965f2df0c636ef9d94e2a830e34578489d0a57 ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.10.0%2Bxpu-cp311-cp311-linux_x86_64.whl#sha256=71ad2f82da0f41eaec159f39fc85854e27c2391efa91b373e550648a6f4aaad3 ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.10.0%2Bxpu-cp312-cp312-linux_x86_64.whl#sha256=b473571d478912f92881cc13f15fa18f8463fb0fb8a068c96ed47a7d45a4da0a ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.10.0%2Bxpu-cp313-cp313-linux_x86_64.whl#sha256=3bc64a746ff25a93de140902c60c9e819d7413f5cea1e88d80999c27a5901e9c ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.10.0%2Bxpu-cp310-cp310-win_amd64.whl#sha256=ce50691ab3fb6301d9b7bb8b3834cf5fa7152a2b5f91fd24c5efdc601a25b780 ; sys_platform == 'win32' and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.10.0%2Bxpu-cp311-cp311-win_amd64.whl#sha256=cb9d37f21cb9fb7df67d62863f021c3144e8d8832b9ea8e8523ac308bc620ea1 ; sys_platform == 'win32' and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.10.0%2Bxpu-cp312-cp312-win_amd64.whl#sha256=3ad605be4728b6d3a28a44d07dd794b1a9e45551b0057815bf25eb2a6d6a56a7 ; sys_platform == 'win32' and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torch @ https://download.pytorch.org/whl/xpu/torch-2.10.0%2Bxpu-cp313-cp313-win_amd64.whl#sha256=2b4b56dd6c792aef82006904fa888692e3782e4ae5da27526801bad4898f05a5 ; sys_platform == 'win32' and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - - "bitsandbytes @ https://github.com/bitsandbytes-foundation/bitsandbytes/releases/download/continuous-release_main/bitsandbytes-1.33.7.preview-py3-none-manylinux_2_24_x86_64.whl ; ('linux' in sys_platform) and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "bitsandbytes @ https://github.com/bitsandbytes-foundation/bitsandbytes/releases/download/continuous-release_main/bitsandbytes-1.33.7.preview-py3-none-win_amd64.whl ; (sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.25.0%2Bxpu-cp310-cp310-manylinux_2_28_x86_64.whl#sha256=7e1e7b170fcf7161c8499b67156c5a05462243626dc0974010791a0bab4378d3 ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.25.0%2Bxpu-cp311-cp311-manylinux_2_28_x86_64.whl#sha256=bd6add201bd7628af70437292e1447abb368e0b5f4ff9abd334ae435efd44792 ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.25.0%2Bxpu-cp312-cp312-manylinux_2_28_x86_64.whl#sha256=6ad2543496bc29e59d3dd614a94d09aa9870318aedb66045344fffddfedd2cf8 ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.25.0%2Bxpu-cp313-cp313-manylinux_2_28_x86_64.whl#sha256=80269f37865fcd8b57f20e4786efae2200bfa2b2727926c3c7acc82f0e7d3548 ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.25.0%2Bxpu-cp310-cp310-win_amd64.whl#sha256=6b9485ba85dcba4d196d6134d9c3332fb228fb2556416bf0450a64e8a472fcba ; sys_platform == 'win32' and python_version == '3.10' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.25.0%2Bxpu-cp311-cp311-win_amd64.whl#sha256=36cbaedf10f6412af5c89afd9aeea474e6a56a0050348ada8fabe1ecaf6b879e ; sys_platform == 'win32' and python_version == '3.11' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.25.0%2Bxpu-cp312-cp312-win_amd64.whl#sha256=738357d97468d75fe3d510ac37e65130f2787f81d9bbc1518898f7396dc3403f ; sys_platform == 'win32' and python_version == '3.12' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", - "torchvision @ https://download.pytorch.org/whl/xpu/torchvision-0.25.0%2Bxpu-cp313-cp313-win_amd64.whl#sha256=1c4b44b36a557f7381e3076fb8843366742238648441d607c8d049c6da0f8886 ; sys_platform == 'win32' and python_version == '3.13' and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", -] -intel-gpu-torch210 = [ - "unsloth[intelgputorch210]" -] -intel = [ - "unsloth[intelgputorch280]", -] -amd = [ - "unsloth[huggingfacenotorch]", - "bitsandbytes>=0.49.1 ; ('linux' in sys_platform) and (platform_machine == 'AMD64' or platform_machine == 'x86_64' or platform_machine == 'aarch64')", - "bitsandbytes>=0.49.1 ; (sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", -] -rocm702-torch280 = [ - "unsloth[amd]", - - "triton @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.0.2/triton-3.4.0%2Brocm7.0.2.gitf9e5bf54-cp311-cp311-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", - "triton @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.0.2/triton-3.4.0%2Brocm7.0.2.gitf9e5bf54-cp312-cp312-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", - "triton @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.0.2/triton-3.4.0%2Brocm7.0.2.gitf9e5bf54-cp313-cp313-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", - - "torch @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.0.2/torch-2.8.0%2Brocm7.0.2.lw.git245bf6ed-cp311-cp311-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", - "torch @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.0.2/torch-2.8.0%2Brocm7.0.2.lw.git245bf6ed-cp312-cp312-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", - "torch @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.0.2/torch-2.8.0%2Brocm7.0.2.lw.git245bf6ed-cp313-cp313-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", - - "torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.0.2/torchvision-0.23.0%2Brocm7.0.2.git824e8c87-cp311-cp311-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", - "torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.0.2/torchvision-0.23.0%2Brocm7.0.2.git824e8c87-cp312-cp312-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", - "torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.0.2/torchvision-0.23.0%2Brocm7.0.2.git824e8c87-cp313-cp313-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", -] -rocm72-torch291 = [ - "unsloth[amd]", - - "triton @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/triton-3.5.1%2Brocm7.2.0.gita272dfa8-cp310-cp310-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", - "triton @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/triton-3.5.1%2Brocm7.2.0.gita272dfa8-cp311-cp311-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", - "triton @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/triton-3.5.1%2Brocm7.2.0.gita272dfa8-cp312-cp312-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", - "triton @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/triton-3.5.1%2Brocm7.2.0.gita272dfa8-cp313-cp313-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", - - "torch @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/torch-2.9.1%2Brocm7.2.0.lw.git7e1940d4-cp310-cp310-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", - "torch @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/torch-2.9.1%2Brocm7.2.0.lw.git7e1940d4-cp311-cp311-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", - "torch @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/torch-2.9.1%2Brocm7.2.0.lw.git7e1940d4-cp312-cp312-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", - "torch @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/torch-2.9.1%2Brocm7.2.0.lw.git7e1940d4-cp313-cp313-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", - "torch @ https://repo.radeon.com/rocm/windows/rocm-rel-7.2/torch-2.9.1%2Brocmsdk20260116-cp312-cp312-win_amd64.whl ; sys_platform == 'win32' and python_version == '3.12'", - - "torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/torchvision-0.24.0%2Brocm7.2.0.gitb919bd0c-cp310-cp310-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", - "torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/torchvision-0.24.0%2Brocm7.2.0.gitb919bd0c-cp311-cp311-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", - "torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/torchvision-0.24.0%2Brocm7.2.0.gitb919bd0c-cp312-cp312-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", - "torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/torchvision-0.24.0%2Brocm7.2.0.gitb919bd0c-cp313-cp313-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", - "torchvision @ https://repo.radeon.com/rocm/windows/rocm-rel-7.2/torchvision-0.24.1%2Brocmsdk20260116-cp312-cp312-win_amd64.whl ; sys_platform == 'win32' and python_version == '3.12'", -] -rocm711-torch291 = [ - "unsloth[amd]", - - "triton @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/triton-3.5.1%2Brocm7.1.1.gita272dfa8-cp310-cp310-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", - "triton @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/triton-3.5.1%2Brocm7.1.1.gita272dfa8-cp311-cp311-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", - "triton @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/triton-3.5.1%2Brocm7.1.1.gita272dfa8-cp312-cp312-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", - "triton @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/triton-3.5.1%2Brocm7.1.1.gita272dfa8-cp313-cp313-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", - - "torch @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/torch-2.9.1%2Brocm7.1.1.lw.git351ff442-cp310-cp310-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", - "torch @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/torch-2.9.1%2Brocm7.1.1.lw.git351ff442-cp311-cp311-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", - "torch @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/torch-2.9.1%2Brocm7.1.1.lw.git351ff442-cp312-cp312-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", - "torch @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/torch-2.9.1%2Brocm7.1.1.lw.git351ff442-cp313-cp313-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", - - "torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/torchvision-0.24.0%2Brocm7.1.1.gitb919bd0c-cp310-cp310-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", - "torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/torchvision-0.24.0%2Brocm7.1.1.gitb919bd0c-cp311-cp311-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", - "torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/torchvision-0.24.0%2Brocm7.1.1.gitb919bd0c-cp312-cp312-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", - "torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/torchvision-0.24.0%2Brocm7.1.1.gitb919bd0c-cp313-cp313-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", -] -rocm72-torch2100 = [ - "unsloth[amd]", - - "triton @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/triton-3.6.0%2Brocm7.2.0.gitba5c1517-cp310-cp310-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", - "triton @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/triton-3.6.0%2Brocm7.2.0.gitba5c1517-cp311-cp311-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", - "triton @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/triton-3.6.0%2Brocm7.2.0.gitba5c1517-cp312-cp312-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", - "triton @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/triton-3.6.0%2Brocm7.2.0.gitba5c1517-cp313-cp313-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", - - "torch @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/torch-2.10.0%2Brocm7.2.0.lw.gitb6ee5fde-cp310-cp310-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", - "torch @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/torch-2.10.0%2Brocm7.2.0.lw.gitb6ee5fde-cp311-cp311-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", - "torch @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/torch-2.10.0%2Brocm7.2.0.lw.gitb6ee5fde-cp312-cp312-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", - "torch @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/torch-2.10.0%2Brocm7.2.0.lw.gitb6ee5fde-cp313-cp313-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", - - "torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/torchvision-0.25.0%2Brocm7.2.0.git82df5f59-cp310-cp310-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", - "torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/torchvision-0.25.0%2Brocm7.2.0.git82df5f59-cp311-cp311-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", - "torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/torchvision-0.25.0%2Brocm7.2.0.git82df5f59-cp312-cp312-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", - "torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.2/torchvision-0.25.0%2Brocm7.2.0.git82df5f59-cp313-cp313-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", -] -rocm711-torch2100 = [ - "unsloth[amd]", - - "triton @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/triton-3.6.0%2Brocm7.1.1.gitba5c1517-cp310-cp310-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", - "triton @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/triton-3.6.0%2Brocm7.1.1.gitba5c1517-cp311-cp311-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", - "triton @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/triton-3.6.0%2Brocm7.1.1.gitba5c1517-cp312-cp312-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", - "triton @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/triton-3.6.0%2Brocm7.1.1.gitba5c1517-cp313-cp313-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", - - "torch @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/torch-2.10.0%2Brocm7.1.1.lw.gitd9556b05-cp310-cp310-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", - "torch @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/torch-2.10.0%2Brocm7.1.1.lw.gitd9556b05-cp311-cp311-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", - "torch @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/torch-2.10.0%2Brocm7.1.1.lw.gitd9556b05-cp312-cp312-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", - "torch @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/torch-2.10.0%2Brocm7.1.1.lw.gitd9556b05-cp313-cp313-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", - - "torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/torchvision-0.25.0%2Brocm7.1.1.git82df5f59-cp310-cp310-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'", - "torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/torchvision-0.25.0%2Brocm7.1.1.git82df5f59-cp311-cp311-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'", - "torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/torchvision-0.25.0%2Brocm7.1.1.git82df5f59-cp312-cp312-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'", - "torchvision @ https://repo.radeon.com/rocm/manylinux/rocm-rel-7.1.1/torchvision-0.25.0%2Brocm7.1.1.git82df5f59-cp313-cp313-linux_x86_64.whl ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'", -] [project.urls] homepage = "http://www.unsloth.ai" documentation = "https://github.com/unslothai/unsloth" repository = "https://github.com/unslothai/unsloth" - -[tool.ruff] -target-version = "py311" -force-exclude = true -extend-exclude = [ - "*chat_templates.py", - "*ollama_template_mappers.py", - "*_auto_install.py", - "*mapper.py", -] - -[tool.ruff.lint] -select = ["E9", "F63", "F7", "F82"] -ignore = [ - "E402", - "E722", - "F403", - "F405", - "F811", - "F821", - "F841", - "F401", - "E731", - "E741", - "F601", - "E712", -] - -[tool.ruff.format] From ddf6f6d1f93a6edc1bfe57082edd1b8172123995 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 24 Mar 2026 07:10:26 -0700 Subject: [PATCH 002/132] fix(studio): remove litellm dep (quarantined on PyPI) (#4553) litellm has been quarantined on PyPI due to a supply chain attack in version 1.82.8 (malicious credential-stealing .pth file). No versions are currently installable, which blocks `unsloth studio setup` at step 8/11 (data-designer deps). Remove litellm from the single-env data-designer requirements so setup completes. litellm can be re-added once PyPI lifts the quarantine. Ref: https://github.com/BerriAI/litellm/issues/24512 --- studio/backend/requirements/single-env/data-designer-deps.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/studio/backend/requirements/single-env/data-designer-deps.txt b/studio/backend/requirements/single-env/data-designer-deps.txt index e0b3d8b72a..9cd0db99e4 100644 --- a/studio/backend/requirements/single-env/data-designer-deps.txt +++ b/studio/backend/requirements/single-env/data-designer-deps.txt @@ -8,7 +8,6 @@ httpx-retries<1,>=0.4.2 json-repair<1,>=0.48.0 jsonpath-rust-bindings<2,>=1.0 jsonschema<5,>=4.0.0 -litellm<1.80.12,>=1.73.6 lxml<7,>=6.0.2 marko<3,>=2.1.2 networkx<4,>=3.0 From 8b77451e75973a042116c38045caeb59cda7b950 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 25 Mar 2026 07:31:31 -0700 Subject: [PATCH 003/132] Update pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index afa96bbbcf..e788cbd23d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,7 @@ dependencies = [ "unsloth_zoo>=2026.3.4", "wheel>=0.42.0", "packaging", - "torch>=2.4.0", + "torch>=2.4.0,<2.11.0", "torchvision", "numpy", "tqdm", From 1c608e8ff78e8cbaff93d1e6e97716067d3e70d8 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 25 Mar 2026 16:13:30 +0000 Subject: [PATCH 004/132] Fix Colab dep install: relax == pins to >= to avoid breaking transformers studio.txt pins huggingface-hub==0.36.2 and datasets==4.3.0 which overwrite Colab's pre-installed versions and break its bundled transformers (is_offline_mode was removed in newer huggingface-hub). Relax == to >= in both colab.py and setup.sh Colab paths so pip keeps existing compatible versions instead of force-upgrading. --- studio/backend/colab.py | 19 ++++++++++++++++++- studio/setup.sh | 7 +++++-- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/studio/backend/colab.py b/studio/backend/colab.py index ad9a7e98b2..53074066a7 100644 --- a/studio/backend/colab.py +++ b/studio/backend/colab.py @@ -30,15 +30,32 @@ def _pip_install_backend_deps() -> None: Used on Colab when the Studio venv does not exist (install.sh was not run). Reads the requirements from studio.txt next to this file. + + Strict ``==`` version pins are relaxed to ``>=`` so we do not clobber + Colab's pre-installed packages (e.g. huggingface-hub, datasets) with + versions that are incompatible with its bundled transformers. """ + import re import subprocess req_file = Path(__file__).parent / "requirements" / "studio.txt" if not req_file.exists(): return + + packages = [] + for line in req_file.read_text().splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + # Relax exact pins (==) to >= so pip keeps existing compatible versions + line = re.sub(r"==", ">=", line) + packages.append(line) + + if not packages: + return print("Installing Studio backend dependencies ...") subprocess.check_call( - [sys.executable, "-m", "pip", "install", "-q", "-r", str(req_file)], + [sys.executable, "-m", "pip", "install", "-q"] + packages, ) diff --git a/studio/setup.sh b/studio/setup.sh index dd758e49e8..682b5cb01a 100755 --- a/studio/setup.sh +++ b/studio/setup.sh @@ -295,9 +295,12 @@ VENV_T5_DIR="$STUDIO_HOME/.venv_t5" _COLAB_NO_VENV=false if [ ! -x "$VENV_DIR/bin/python" ]; then if [ "$IS_COLAB" = true ]; then - # On Colab there is no Studio venv -- install backend deps into system Python + # On Colab there is no Studio venv -- install backend deps into system Python. + # Relax strict == pins to >= so we don't clobber Colab's pre-installed + # packages (huggingface-hub, datasets) with incompatible versions. echo " Colab detected, installing Studio backend dependencies..." - pip install -q -r "$SCRIPT_DIR/backend/requirements/studio.txt" 2>/dev/null || true + sed 's/==/>=/' "$SCRIPT_DIR/backend/requirements/studio.txt" \ + | pip install -q -r /dev/stdin 2>/dev/null || true _COLAB_NO_VENV=true else echo "❌ ERROR: Virtual environment not found at $VENV_DIR" From 0bb6379aad1aab99e150180ce722128a4d4a471d Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 25 Mar 2026 16:15:29 +0000 Subject: [PATCH 005/132] Fix Colab huggingface-hub conflict and pip bootstrap on uv venvs - colab.py / setup.sh: relax == pins to >= when installing studio.txt on Colab so huggingface-hub 0.36.2 does not clobber Colab's bundled version (which breaks transformers is_offline_mode import) - install_python_stack.py: when uv is unavailable and pip is missing (uv-created venvs), bootstrap via ensurepip before attempting upgrade - Bump version to 2026.3.14 --- studio/install_python_stack.py | 23 +++++++++++++++++++---- unsloth/models/_utils.py | 2 +- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/studio/install_python_stack.py b/studio/install_python_stack.py index 39fec2e6f5..603ac162c9 100644 --- a/studio/install_python_stack.py +++ b/studio/install_python_stack.py @@ -373,10 +373,25 @@ def install_python_stack() -> int: ], ) else: - run( - "Upgrading pip", - [sys.executable, "-m", "pip", "install", "--upgrade", "pip"], - ) + # pip may not exist yet (uv-created venvs omit it). Try ensurepip + # first, then upgrade. Only fall back to a direct upgrade when pip + # is already present. + _has_pip = subprocess.run( + [sys.executable, "-m", "pip", "--version"], + stdout = subprocess.DEVNULL, + stderr = subprocess.DEVNULL, + ).returncode == 0 + + if not _has_pip: + run( + "Bootstrapping pip via ensurepip", + [sys.executable, "-m", "ensurepip", "--upgrade"], + ) + else: + run( + "Upgrading pip", + [sys.executable, "-m", "pip", "install", "--upgrade", "pip"], + ) # 3. Core packages: unsloth-zoo + unsloth (or custom package name) if skip_base: diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 1fffc62f81..da912dec76 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2026.3.13" +__version__ = "2026.3.14" __all__ = [ "SUPPORTS_BFLOAT16", From d9d1a63397c6a94851dfc848c04765df4d687031 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 25 Mar 2026 16:36:25 +0000 Subject: [PATCH 006/132] Fix Colab huggingface-hub conflict, ensurepip fallback, bump to 2026.3.14 - Strip version constraints on Colab dep install - Upgrade huggingface-hub>=1.0 if is_offline_mode is missing - ensurepip fallback for uv venvs without pip - Bump installer pins to 2026.3.14 --- install.ps1 | 6 +++--- install.sh | 6 +++--- studio/backend/colab.py | 24 ++++++++++++++++++------ studio/setup.sh | 8 +++++--- 4 files changed, 29 insertions(+), 15 deletions(-) diff --git a/install.ps1 b/install.ps1 index 86e4288e8e..3ef251715e 100644 --- a/install.ps1 +++ b/install.ps1 @@ -607,7 +607,7 @@ shell.Run cmd, 0, False # Migrated env: force-reinstall unsloth+unsloth-zoo to ensure clean state # in the new venv location, while preserving existing torch/CUDA Write-Host "==> Upgrading unsloth in migrated environment..." - uv pip install --python $VenvPython --reinstall-package unsloth --reinstall-package unsloth-zoo "unsloth>=2026.3.12" unsloth-zoo + uv pip install --python $VenvPython --reinstall-package unsloth --reinstall-package unsloth-zoo "unsloth>=2026.3.14" unsloth-zoo if ($StudioLocalInstall) { Write-Host "==> Overlaying local repo (editable)..." uv pip install --python $VenvPython -e $RepoRoot --no-deps @@ -622,7 +622,7 @@ shell.Run cmd, 0, False Write-Host "==> Installing unsloth (this may take a few minutes)..." if ($StudioLocalInstall) { - uv pip install --python $VenvPython --upgrade-package unsloth "unsloth>=2026.3.12" unsloth-zoo + uv pip install --python $VenvPython --upgrade-package unsloth "unsloth>=2026.3.14" unsloth-zoo Write-Host "==> Overlaying local repo (editable)..." uv pip install --python $VenvPython -e $RepoRoot --no-deps } else { @@ -632,7 +632,7 @@ shell.Run cmd, 0, False # Fallback: GPU detection failed to produce a URL -- let uv resolve torch Write-Host "==> Installing unsloth (this may take a few minutes)..." if ($StudioLocalInstall) { - uv pip install --python $VenvPython unsloth-zoo "unsloth>=2026.3.12" --torch-backend=auto + uv pip install --python $VenvPython unsloth-zoo "unsloth>=2026.3.14" --torch-backend=auto Write-Host "==> Overlaying local repo (editable)..." uv pip install --python $VenvPython -e $RepoRoot --no-deps } else { diff --git a/install.sh b/install.sh index ebb5b1f9ae..a0f6ef2ee5 100755 --- a/install.sh +++ b/install.sh @@ -767,7 +767,7 @@ if [ "$_MIGRATED" = true ]; then echo "==> Upgrading unsloth in migrated environment..." uv pip install --python "$_VENV_PY" \ --reinstall-package unsloth --reinstall-package unsloth-zoo \ - "unsloth>=2026.3.12" unsloth-zoo + "unsloth>=2026.3.14" unsloth-zoo if [ "$STUDIO_LOCAL_INSTALL" = true ]; then echo "==> Overlaying local repo (editable)..." uv pip install --python "$_VENV_PY" -e "$_REPO_ROOT" --no-deps @@ -781,7 +781,7 @@ elif [ -n "$TORCH_INDEX_URL" ]; then echo "==> Installing unsloth (this may take a few minutes)..." if [ "$STUDIO_LOCAL_INSTALL" = true ]; then uv pip install --python "$_VENV_PY" \ - --upgrade-package unsloth "unsloth>=2026.3.12" unsloth-zoo + --upgrade-package unsloth "unsloth>=2026.3.14" unsloth-zoo echo "==> Overlaying local repo (editable)..." uv pip install --python "$_VENV_PY" -e "$_REPO_ROOT" --no-deps else @@ -792,7 +792,7 @@ else # Fallback: GPU detection failed to produce a URL -- let uv resolve torch echo "==> Installing unsloth (this may take a few minutes)..." if [ "$STUDIO_LOCAL_INSTALL" = true ]; then - uv pip install --python "$_VENV_PY" unsloth-zoo "unsloth>=2026.3.12" --torch-backend=auto + uv pip install --python "$_VENV_PY" unsloth-zoo "unsloth>=2026.3.14" --torch-backend=auto echo "==> Overlaying local repo (editable)..." uv pip install --python "$_VENV_PY" -e "$_REPO_ROOT" --no-deps else diff --git a/studio/backend/colab.py b/studio/backend/colab.py index 53074066a7..a6fac0d413 100644 --- a/studio/backend/colab.py +++ b/studio/backend/colab.py @@ -31,9 +31,9 @@ def _pip_install_backend_deps() -> None: Used on Colab when the Studio venv does not exist (install.sh was not run). Reads the requirements from studio.txt next to this file. - Strict ``==`` version pins are relaxed to ``>=`` so we do not clobber - Colab's pre-installed packages (e.g. huggingface-hub, datasets) with - versions that are incompatible with its bundled transformers. + Version constraints are stripped entirely so pip keeps whatever Colab + already has installed (e.g. huggingface-hub, datasets, transformers) + and only installs genuinely missing packages like structlog, fastapi. """ import re import subprocess @@ -47,9 +47,10 @@ def _pip_install_backend_deps() -> None: line = line.strip() if not line or line.startswith("#"): continue - # Relax exact pins (==) to >= so pip keeps existing compatible versions - line = re.sub(r"==", ">=", line) - packages.append(line) + # Strip all version constraints -- just keep the package name + pkg_name = re.split(r"[><=!~;\[]", line)[0].strip() + if pkg_name: + packages.append(pkg_name) if not packages: return @@ -58,6 +59,17 @@ def _pip_install_backend_deps() -> None: [sys.executable, "-m", "pip", "install", "-q"] + packages, ) + # Colab ships huggingface-hub 0.36.x which removed is_offline_mode, + # breaking transformers. Upgrade to 1.0+ which restored it. + try: + from huggingface_hub import is_offline_mode # noqa: F401 + except ImportError: + print("Upgrading huggingface-hub (is_offline_mode missing) ...") + subprocess.check_call( + [sys.executable, "-m", "pip", "install", "-q", + "huggingface-hub>=1.0"], + ) + def _bootstrap_studio_venv() -> None: """Expose the Studio venv's site-packages to the current interpreter. diff --git a/studio/setup.sh b/studio/setup.sh index 682b5cb01a..f103dd95a4 100755 --- a/studio/setup.sh +++ b/studio/setup.sh @@ -296,10 +296,12 @@ _COLAB_NO_VENV=false if [ ! -x "$VENV_DIR/bin/python" ]; then if [ "$IS_COLAB" = true ]; then # On Colab there is no Studio venv -- install backend deps into system Python. - # Relax strict == pins to >= so we don't clobber Colab's pre-installed - # packages (huggingface-hub, datasets) with incompatible versions. + # Strip all version constraints so pip keeps Colab's pre-installed + # packages (huggingface-hub, datasets, transformers) and only pulls + # in genuinely missing ones (structlog, fastapi, etc.). echo " Colab detected, installing Studio backend dependencies..." - sed 's/==/>=/' "$SCRIPT_DIR/backend/requirements/studio.txt" \ + sed 's/[><=!~;].*//' "$SCRIPT_DIR/backend/requirements/studio.txt" \ + | grep -v '^#' | grep -v '^$' \ | pip install -q -r /dev/stdin 2>/dev/null || true _COLAB_NO_VENV=true else From e568000a925da5c189407ce106bc2d0f9dff45d1 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 27 Mar 2026 07:23:42 -0700 Subject: [PATCH 007/132] Update pyproject.toml --- pyproject.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e788cbd23d..97096dc9e5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ classifiers = [ "Topic :: Scientific/Engineering :: Artificial Intelligence", ] dependencies = [ - "unsloth_zoo>=2026.3.4", + "unsloth_zoo>=2026.3.6", "wheel>=0.42.0", "packaging", "torch>=2.4.0,<2.11.0", @@ -85,7 +85,7 @@ exclude = ["images*", "tests*", "kernels/moe*"] [project.optional-dependencies] huggingface = [ - "unsloth_zoo>=2026.3.4", + "unsloth_zoo>=2026.3.6", "wheel>=0.42.0", "packaging", "torchvision", @@ -473,7 +473,7 @@ colab-ampere-torch220 = [ "unsloth[flashattention]", ] colab-new = [ - "unsloth_zoo>=2026.3.4", + "unsloth_zoo>=2026.3.6", "packaging", "tyro", "transformers>=4.51.3,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1,!=4.57.0,!=4.57.4,!=4.57.5,!=5.0.0,!=5.1.0,<=5.3.0", From a728f7c308941ff64ff90567f44c723235d92d52 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 31 Mar 2026 06:51:58 -0700 Subject: [PATCH 008/132] Update pyproject.toml --- pyproject.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 97096dc9e5..66f385aabb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ classifiers = [ "Topic :: Scientific/Engineering :: Artificial Intelligence", ] dependencies = [ - "unsloth_zoo>=2026.3.6", + "unsloth_zoo>=2026.3.7", "wheel>=0.42.0", "packaging", "torch>=2.4.0,<2.11.0", @@ -85,7 +85,7 @@ exclude = ["images*", "tests*", "kernels/moe*"] [project.optional-dependencies] huggingface = [ - "unsloth_zoo>=2026.3.6", + "unsloth_zoo>=2026.3.7", "wheel>=0.42.0", "packaging", "torchvision", @@ -473,7 +473,7 @@ colab-ampere-torch220 = [ "unsloth[flashattention]", ] colab-new = [ - "unsloth_zoo>=2026.3.6", + "unsloth_zoo>=2026.3.7", "packaging", "tyro", "transformers>=4.51.3,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1,!=4.57.0,!=4.57.4,!=4.57.5,!=5.0.0,!=5.1.0,<=5.3.0", From 396aa05ead45d3bc03f545315c10881a68b67e32 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 2 Apr 2026 12:03:10 -0700 Subject: [PATCH 009/132] Update pyproject.toml --- pyproject.toml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 66f385aabb..d4a5fc3ff5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ classifiers = [ "Topic :: Scientific/Engineering :: Artificial Intelligence", ] dependencies = [ - "unsloth_zoo>=2026.3.7", + "unsloth_zoo>=2026.4.2", "wheel>=0.42.0", "packaging", "torch>=2.4.0,<2.11.0", @@ -46,7 +46,7 @@ dependencies = [ "huggingface_hub>=0.34.0", "hf_transfer", "diffusers", - "transformers>=4.51.3,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1,!=4.57.0,!=4.57.4,!=4.57.5,!=5.0.0,!=5.1.0,<=5.3.0", + "transformers>=4.51.3,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1,!=4.57.0,!=4.57.4,!=4.57.5,!=5.0.0,!=5.1.0,<=5.5.0", "trl>=0.18.2,!=0.19.0,<=0.24.0", "typer", "pydantic", @@ -85,7 +85,7 @@ exclude = ["images*", "tests*", "kernels/moe*"] [project.optional-dependencies] huggingface = [ - "unsloth_zoo>=2026.3.7", + "unsloth_zoo>=2026.4.2", "wheel>=0.42.0", "packaging", "torchvision", @@ -101,7 +101,7 @@ huggingface = [ "huggingface_hub>=0.34.0", "hf_transfer", "diffusers", - "transformers>=4.51.3,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1,!=4.57.0,!=4.57.4,!=4.57.5,!=5.0.0,!=5.1.0,<=5.3.0", + "transformers>=4.51.3,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1,!=4.57.0,!=4.57.4,!=4.57.5,!=5.0.0,!=5.1.0,<=5.5.0", "trl>=0.18.2,!=0.19.0,<=0.24.0", "typer", "pydantic", @@ -473,10 +473,10 @@ colab-ampere-torch220 = [ "unsloth[flashattention]", ] colab-new = [ - "unsloth_zoo>=2026.3.7", + "unsloth_zoo>=2026.4.2", "packaging", "tyro", - "transformers>=4.51.3,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1,!=4.57.0,!=4.57.4,!=4.57.5,!=5.0.0,!=5.1.0,<=5.3.0", + "transformers>=4.51.3,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1,!=4.57.0,!=4.57.4,!=4.57.5,!=5.0.0,!=5.1.0,<=5.5.0", "datasets>=3.4.1,!=4.0.*,!=4.1.0,<4.4.0", "sentencepiece>=0.2.0", "tqdm", From b2580ae32bfba1c5caa3cd67427aad024adfd0a9 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 6 Apr 2026 09:21:23 -0700 Subject: [PATCH 010/132] Update pyproject.toml --- pyproject.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d4a5fc3ff5..2c9ec4466a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ classifiers = [ "Topic :: Scientific/Engineering :: Artificial Intelligence", ] dependencies = [ - "unsloth_zoo>=2026.4.2", + "unsloth_zoo>=2026.4.3", "wheel>=0.42.0", "packaging", "torch>=2.4.0,<2.11.0", @@ -85,7 +85,7 @@ exclude = ["images*", "tests*", "kernels/moe*"] [project.optional-dependencies] huggingface = [ - "unsloth_zoo>=2026.4.2", + "unsloth_zoo>=2026.4.3", "wheel>=0.42.0", "packaging", "torchvision", @@ -473,7 +473,7 @@ colab-ampere-torch220 = [ "unsloth[flashattention]", ] colab-new = [ - "unsloth_zoo>=2026.4.2", + "unsloth_zoo>=2026.4.3", "packaging", "tyro", "transformers>=4.51.3,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1,!=4.57.0,!=4.57.4,!=4.57.5,!=5.0.0,!=5.1.0,<=5.5.0", From e8355451eae64f2ae7e43c95ff712c97f4507a11 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 15 Apr 2026 08:07:33 -0700 Subject: [PATCH 011/132] Update pyproject.toml --- pyproject.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2c9ec4466a..e11e070cd9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ classifiers = [ "Topic :: Scientific/Engineering :: Artificial Intelligence", ] dependencies = [ - "unsloth_zoo>=2026.4.3", + "unsloth_zoo>=2026.4.7", "wheel>=0.42.0", "packaging", "torch>=2.4.0,<2.11.0", @@ -85,7 +85,7 @@ exclude = ["images*", "tests*", "kernels/moe*"] [project.optional-dependencies] huggingface = [ - "unsloth_zoo>=2026.4.3", + "unsloth_zoo>=2026.4.7", "wheel>=0.42.0", "packaging", "torchvision", @@ -473,7 +473,7 @@ colab-ampere-torch220 = [ "unsloth[flashattention]", ] colab-new = [ - "unsloth_zoo>=2026.4.3", + "unsloth_zoo>=2026.4.7", "packaging", "tyro", "transformers>=4.51.3,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1,!=4.57.0,!=4.57.4,!=4.57.5,!=5.0.0,!=5.1.0,<=5.5.0", From 0c24d61708d62d318d8106fde8dc531c90dc5267 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 16 Apr 2026 12:06:55 -0700 Subject: [PATCH 012/132] Update pyproject.toml --- pyproject.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e11e070cd9..540520de9f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ classifiers = [ "Topic :: Scientific/Engineering :: Artificial Intelligence", ] dependencies = [ - "unsloth_zoo>=2026.4.7", + "unsloth_zoo>=2026.4.8", "wheel>=0.42.0", "packaging", "torch>=2.4.0,<2.11.0", @@ -85,7 +85,7 @@ exclude = ["images*", "tests*", "kernels/moe*"] [project.optional-dependencies] huggingface = [ - "unsloth_zoo>=2026.4.7", + "unsloth_zoo>=2026.4.8", "wheel>=0.42.0", "packaging", "torchvision", @@ -473,7 +473,7 @@ colab-ampere-torch220 = [ "unsloth[flashattention]", ] colab-new = [ - "unsloth_zoo>=2026.4.7", + "unsloth_zoo>=2026.4.8", "packaging", "tyro", "transformers>=4.51.3,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1,!=4.57.0,!=4.57.4,!=4.57.5,!=5.0.0,!=5.1.0,<=5.5.0", From 2e3b2bdc2716ca3311b48ef3d27baa631a2180ec Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 5 May 2026 05:27:54 -0700 Subject: [PATCH 013/132] Update pyproject.toml --- pyproject.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 540520de9f..31281ca754 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ classifiers = [ "Topic :: Scientific/Engineering :: Artificial Intelligence", ] dependencies = [ - "unsloth_zoo>=2026.4.8", + "unsloth_zoo>=2026.5.1", "wheel>=0.42.0", "packaging", "torch>=2.4.0,<2.11.0", @@ -85,7 +85,7 @@ exclude = ["images*", "tests*", "kernels/moe*"] [project.optional-dependencies] huggingface = [ - "unsloth_zoo>=2026.4.8", + "unsloth_zoo>=2026.5.1", "wheel>=0.42.0", "packaging", "torchvision", @@ -473,7 +473,7 @@ colab-ampere-torch220 = [ "unsloth[flashattention]", ] colab-new = [ - "unsloth_zoo>=2026.4.8", + "unsloth_zoo>=2026.5.1", "packaging", "tyro", "transformers>=4.51.3,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1,!=4.57.0,!=4.57.4,!=4.57.5,!=5.0.0,!=5.1.0,<=5.5.0", From 41abe8904142493acabad913f44f1f7ffd9b82b9 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Wed, 6 May 2026 00:25:04 +0000 Subject: [PATCH 014/132] Bump version to 2026.5.2 Cuts a new PyPI release that ships PR #5296: Studio chat history and attachments work again with newer @assistant-ui/react, plus the pinned assistant-ui surface and frontend package-lock.json so future installs cannot drift back onto a broken bundle. --- unsloth/models/_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 1120e1941d..34fec53504 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2026.5.1" +__version__ = "2026.5.2" __all__ = [ "SUPPORTS_BFLOAT16", From 2c4bf9ae358a80cd2e4020d33134d3816a9b6498 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 5 May 2026 23:22:22 -0700 Subject: [PATCH 015/132] Route CPU-only Linux x86_64 to ggml-org/llama.cpp prebuilts (#5302) * Route CPU-only Linux x86_64 to ggml-org/llama.cpp prebuilts setup.sh hard-coded _HELPER_RELEASE_REPO=unslothai/llama.cpp for every non-Darwin host. unslothai/llama.cpp only publishes Linux CUDA bundles (app-*-linux-x64-cuda*.tar.gz), so a CPU-only Linux host walked ~30 releases looking for a non-existent app-*-linux-x64-cpu asset, exited the prebuilt planner with "no compatible Linux prebuilt asset was found", and fell through to a source build. Free CI runners (ubuntu-latest with no GPU) hit this on every install, and anyone running Studio on a Linux laptop without an NVIDIA GPU paid the ~3 minute cmake+make cost on first install. ggml-org publishes llama--bin-ubuntu-x64.tar.gz on every release and install_llama_prebuilt.py already knows how to fetch it: when called with --published-repo ggml-org/llama.cpp, the Linux x86_64 + not has_usable_nvidia branch in direct_upstream_release_plan picks up that asset directly. The fix is purely on the routing side. Tighten the gate so a Linux host routes to ggml-org only when it is x86_64 and has no GPU detection tool installed (nvidia-smi, rocminfo, amd-smi, hipconfig, hipinfo). Everything else stays on the current path: - macOS: already on ggml-org, unchanged - Windows: already on ggml-org via setup.ps1, unchanged - Linux CUDA: nvidia-smi present -> unslothai/llama.cpp, unchanged - Linux ROCm: rocminfo / amd-smi / hipconfig / hipinfo present -> unslothai/llama.cpp -> source build with HIP, unchanged - Linux Intel / Vulkan / SYCL: no NVIDIA / AMD tools, hits the new ggml-org route, gets upstream CPU asset (same as today's source-build CPU output, ~3 min faster) - Linux arm64 / s390x: not x86_64 -> unslothai/llama.cpp -> source build, unchanged * Tighten routing comment in studio/setup.sh --- studio/setup.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/studio/setup.sh b/studio/setup.sh index 3e875eed30..1b6a3db343 100755 --- a/studio/setup.sh +++ b/studio/setup.sh @@ -582,11 +582,30 @@ _LLAMA_CPP_DEGRADED=false _LLAMA_FORCE_COMPILE="${UNSLOTH_LLAMA_FORCE_COMPILE:-0}" _REQUESTED_LLAMA_TAG="${UNSLOTH_LLAMA_TAG:-${_DEFAULT_LLAMA_TAG}}" _HOST_SYSTEM="$(uname -s 2>/dev/null || true)" +_HOST_MACHINE="$(uname -m 2>/dev/null || true)" + +# Pick the release repo install_llama_prebuilt.py plans against. +# unslothai/llama.cpp ships only Linux CUDA bundles, so CPU-only Linux +# x86_64 routes to ggml-org for bin-ubuntu-x64.tar.gz. Anything with a +# GPU tool installed stays on unslothai (CUDA bundle / ROCm source build). +_LINUX_HAS_GPU=false +for _GPU_TOOL in nvidia-smi rocminfo amd-smi hipconfig hipinfo; do + if command -v "$_GPU_TOOL" >/dev/null 2>&1; then + _LINUX_HAS_GPU=true + break + fi +done + if [ "$_HOST_SYSTEM" = "Darwin" ]; then _HELPER_RELEASE_REPO="ggml-org/llama.cpp" +elif [ "$_HOST_SYSTEM" = "Linux" ] \ + && [ "$_HOST_MACHINE" = "x86_64" ] \ + && [ "$_LINUX_HAS_GPU" = false ]; then + _HELPER_RELEASE_REPO="ggml-org/llama.cpp" else _HELPER_RELEASE_REPO="unslothai/llama.cpp" fi +unset _GPU_TOOL _LLAMA_PR="${UNSLOTH_LLAMA_PR:-}" _SKIP_PREBUILT_INSTALL=false _LLAMA_PR_FORCE="${UNSLOTH_LLAMA_PR_FORCE:-${_DEFAULT_LLAMA_PR_FORCE}}" From e1fe3be939a1412d6d5832fb3bce7068ea7f1e0f Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 18 May 2026 06:30:03 -0700 Subject: [PATCH 016/132] Update pyproject.toml --- pyproject.toml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 31281ca754..457178baba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ classifiers = [ "Topic :: Scientific/Engineering :: Artificial Intelligence", ] dependencies = [ - "unsloth_zoo>=2026.5.1", + "unsloth_zoo>=2026.5.2", "wheel>=0.42.0", "packaging", "torch>=2.4.0,<2.11.0", @@ -76,16 +76,18 @@ studio = [ "frontend/*.yaml", "frontend/.git*", "backend/requirements/**/*", + "backend/plugins/**/*", "backend/core/data_recipe/oxc-validator/*.json", "backend/core/data_recipe/oxc-validator/*.mjs", ] [tool.setuptools.packages.find] -exclude = ["images*", "tests*", "kernels/moe*"] +include = ["unsloth*", "unsloth_cli*", "studio", "studio.backend*"] +exclude = ["images*", "tests*", "*.node_modules", "*.node_modules.*"] [project.optional-dependencies] huggingface = [ - "unsloth_zoo>=2026.5.1", + "unsloth_zoo>=2026.5.2", "wheel>=0.42.0", "packaging", "torchvision", @@ -473,7 +475,7 @@ colab-ampere-torch220 = [ "unsloth[flashattention]", ] colab-new = [ - "unsloth_zoo>=2026.5.1", + "unsloth_zoo>=2026.5.2", "packaging", "tyro", "transformers>=4.51.3,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1,!=4.57.0,!=4.57.4,!=4.57.5,!=5.0.0,!=5.1.0,<=5.5.0", From d482382a92154492c0ac62f9ef086e1783cc7fa6 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 18 May 2026 06:39:41 -0700 Subject: [PATCH 017/132] Sync pyproject.toml fixes from main into pip (#5555) - Add missing cu124onlytorch240 extras (xformers==0.0.28.post1) - Add sentence-transformers to huggingface and colab-new extras - Add triton extras group (split out from main deps) - Add windows extras group - Drop xformers<0.0.27 cap in colab-no-deps; gate on linux/win + x86_64 - Add [tool.ruff] config (target py311, lint select/ignore, extend-exclude) - Add [tool.pytest.ini_options] testpaths = ["tests/security"] Only PyPI-compatible (version-pin) entries copied; URL-based intel/amd/rocm/ flashattentiontorch* extras from main are intentionally skipped. --- pyproject.toml | 51 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 457178baba..182118a3fd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -86,6 +86,10 @@ include = ["unsloth*", "unsloth_cli*", "studio", "studio.backend*"] exclude = ["images*", "tests*", "*.node_modules", "*.node_modules.*"] [project.optional-dependencies] +triton = [ + "triton>=3.0.0 ; ('linux' in sys_platform)", + "triton-windows ; (sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", +] huggingface = [ "unsloth_zoo>=2026.5.2", "wheel>=0.42.0", @@ -105,11 +109,17 @@ huggingface = [ "diffusers", "transformers>=4.51.3,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1,!=4.57.0,!=4.57.4,!=4.57.5,!=5.0.0,!=5.1.0,<=5.5.0", "trl>=0.18.2,!=0.19.0,<=0.24.0", + "sentence-transformers", "typer", "pydantic", "pyyaml", "nest-asyncio", ] +windows = [ + "unsloth[huggingface]", + "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0 ; (sys_platform == 'win32')", + "xformers>=0.0.22.post7 ; (sys_platform == 'win32')", +] base = [ "unsloth[huggingface]", ] @@ -149,6 +159,9 @@ cu118onlytorch240 = [ cu121onlytorch240 = [ "xformers==0.0.27.post2 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] +cu124onlytorch240 = [ + "xformers==0.0.28.post1 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", +] cu118onlytorch250 = [ "xformers==0.0.28.post2 ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] @@ -490,6 +503,7 @@ colab-new = [ "hf_transfer", "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", "unsloth[triton]", + "sentence-transformers", "typer", "pydantic", "pyyaml", @@ -499,7 +513,7 @@ colab-no-deps = [ "accelerate>=0.34.1", "trl>=0.18.2,!=0.19.0,<=0.24.0", "peft>=0.18.0", - "xformers<0.0.27", + "xformers ; ('linux' in sys_platform or sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", "protobuf", ] @@ -739,3 +753,38 @@ cu130-ampere-torch2100 = [ homepage = "https://unsloth.ai" documentation = "https://unsloth.ai/docs" repository = "https://github.com/unslothai/unsloth" + +[tool.ruff] +target-version = "py311" +force-exclude = true +extend-exclude = [ + "*chat_templates.py", + "*ollama_template_mappers.py", + "*_auto_install.py", + "*mapper.py", +] + +[tool.ruff.lint] +select = ["E9", "F63", "F7", "F82"] +ignore = [ + "E402", + "E722", + "F403", + "F405", + "F811", + "F821", + "F841", + "F401", + "E731", + "E741", + "F601", + "E712", +] + +[tool.ruff.format] + +[tool.pytest.ini_options] +# Narrow the default test discovery so `pytest` from the repo root +# does NOT pick up the GPU-heavy tests under tests/python, tests/qlora, +# etc. The CI security job runs `pytest tests/security` explicitly. +testpaths = ["tests/security"] From 405add94a11dafebd20343ed2d81848f292b1511 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 19 May 2026 07:00:38 -0700 Subject: [PATCH 018/132] Update pyproject.toml --- pyproject.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 182118a3fd..22ab718f9c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ classifiers = [ "Topic :: Scientific/Engineering :: Artificial Intelligence", ] dependencies = [ - "unsloth_zoo>=2026.5.2", + "unsloth_zoo>=2026.5.3", "wheel>=0.42.0", "packaging", "torch>=2.4.0,<2.11.0", @@ -91,7 +91,7 @@ triton = [ "triton-windows ; (sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] huggingface = [ - "unsloth_zoo>=2026.5.2", + "unsloth_zoo>=2026.5.3", "wheel>=0.42.0", "packaging", "torchvision", @@ -488,7 +488,7 @@ colab-ampere-torch220 = [ "unsloth[flashattention]", ] colab-new = [ - "unsloth_zoo>=2026.5.2", + "unsloth_zoo>=2026.5.3", "packaging", "tyro", "transformers>=4.51.3,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1,!=4.57.0,!=4.57.4,!=4.57.5,!=5.0.0,!=5.1.0,<=5.5.0", From e2215c9d11c7398bf4f3a675e9670eb5281ff08e Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 22 May 2026 09:24:40 -0700 Subject: [PATCH 019/132] Update pyproject.toml --- pyproject.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 22ab718f9c..53f2dd7979 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ classifiers = [ "Topic :: Scientific/Engineering :: Artificial Intelligence", ] dependencies = [ - "unsloth_zoo>=2026.5.3", + "unsloth_zoo>=2026.5.4", "wheel>=0.42.0", "packaging", "torch>=2.4.0,<2.11.0", @@ -91,7 +91,7 @@ triton = [ "triton-windows ; (sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] huggingface = [ - "unsloth_zoo>=2026.5.3", + "unsloth_zoo>=2026.5.4", "wheel>=0.42.0", "packaging", "torchvision", @@ -488,7 +488,7 @@ colab-ampere-torch220 = [ "unsloth[flashattention]", ] colab-new = [ - "unsloth_zoo>=2026.5.3", + "unsloth_zoo>=2026.5.4", "packaging", "tyro", "transformers>=4.51.3,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1,!=4.57.0,!=4.57.4,!=4.57.5,!=5.0.0,!=5.1.0,<=5.5.0", From e31d4c6aea86de85cbe33223bd970a5557eae2eb Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sun, 31 May 2026 07:11:09 -0700 Subject: [PATCH 020/132] Update pyproject.toml --- pyproject.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 53f2dd7979..f6094e1c7d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ classifiers = [ "Topic :: Scientific/Engineering :: Artificial Intelligence", ] dependencies = [ - "unsloth_zoo>=2026.5.4", + "unsloth_zoo>=2026.5.5", "wheel>=0.42.0", "packaging", "torch>=2.4.0,<2.11.0", @@ -91,7 +91,7 @@ triton = [ "triton-windows ; (sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] huggingface = [ - "unsloth_zoo>=2026.5.4", + "unsloth_zoo>=2026.5.5", "wheel>=0.42.0", "packaging", "torchvision", @@ -488,7 +488,7 @@ colab-ampere-torch220 = [ "unsloth[flashattention]", ] colab-new = [ - "unsloth_zoo>=2026.5.4", + "unsloth_zoo>=2026.5.5", "packaging", "tyro", "transformers>=4.51.3,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1,!=4.57.0,!=4.57.4,!=4.57.5,!=5.0.0,!=5.1.0,<=5.5.0", From ff1088af40649df52908004bcd4346520eefeb58 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 3 Jun 2026 06:36:36 -0700 Subject: [PATCH 021/132] Update pyproject.toml --- pyproject.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f6094e1c7d..266bab9559 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ classifiers = [ "Topic :: Scientific/Engineering :: Artificial Intelligence", ] dependencies = [ - "unsloth_zoo>=2026.5.5", + "unsloth_zoo>=2026.6.1", "wheel>=0.42.0", "packaging", "torch>=2.4.0,<2.11.0", @@ -91,7 +91,7 @@ triton = [ "triton-windows ; (sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] huggingface = [ - "unsloth_zoo>=2026.5.5", + "unsloth_zoo>=2026.6.1", "wheel>=0.42.0", "packaging", "torchvision", @@ -488,7 +488,7 @@ colab-ampere-torch220 = [ "unsloth[flashattention]", ] colab-new = [ - "unsloth_zoo>=2026.5.5", + "unsloth_zoo>=2026.6.1", "packaging", "tyro", "transformers>=4.51.3,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1,!=4.57.0,!=4.57.4,!=4.57.5,!=5.0.0,!=5.1.0,<=5.5.0", From 43c669e6ec580206660995bf5cf15ca71e7f4c93 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 10 Jun 2026 11:11:36 -0700 Subject: [PATCH 022/132] Update pyproject.toml --- pyproject.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 266bab9559..424266073b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ classifiers = [ "Topic :: Scientific/Engineering :: Artificial Intelligence", ] dependencies = [ - "unsloth_zoo>=2026.6.1", + "unsloth_zoo>=2026.6.2", "wheel>=0.42.0", "packaging", "torch>=2.4.0,<2.11.0", @@ -91,7 +91,7 @@ triton = [ "triton-windows ; (sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] huggingface = [ - "unsloth_zoo>=2026.6.1", + "unsloth_zoo>=2026.6.2", "wheel>=0.42.0", "packaging", "torchvision", @@ -488,7 +488,7 @@ colab-ampere-torch220 = [ "unsloth[flashattention]", ] colab-new = [ - "unsloth_zoo>=2026.6.1", + "unsloth_zoo>=2026.6.2", "packaging", "tyro", "transformers>=4.51.3,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1,!=4.57.0,!=4.57.4,!=4.57.5,!=5.0.0,!=5.1.0,<=5.5.0", From fc9d82f6bfe964aaf25e23f618144cf6e18643c1 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 11 Jun 2026 09:22:35 -0700 Subject: [PATCH 023/132] Update pyproject.toml --- pyproject.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 424266073b..dcd5d91491 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ classifiers = [ "Topic :: Scientific/Engineering :: Artificial Intelligence", ] dependencies = [ - "unsloth_zoo>=2026.6.2", + "unsloth_zoo>=2026.6.3", "wheel>=0.42.0", "packaging", "torch>=2.4.0,<2.11.0", @@ -91,7 +91,7 @@ triton = [ "triton-windows ; (sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] huggingface = [ - "unsloth_zoo>=2026.6.2", + "unsloth_zoo>=2026.6.3", "wheel>=0.42.0", "packaging", "torchvision", @@ -488,7 +488,7 @@ colab-ampere-torch220 = [ "unsloth[flashattention]", ] colab-new = [ - "unsloth_zoo>=2026.6.2", + "unsloth_zoo>=2026.6.3", "packaging", "tyro", "transformers>=4.51.3,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1,!=4.57.0,!=4.57.4,!=4.57.5,!=5.0.0,!=5.1.0,<=5.5.0", From 0e929cddac18a228e832e1268d02a6c567d7acdf Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 12 Jun 2026 06:35:51 -0700 Subject: [PATCH 024/132] Update pyproject.toml --- pyproject.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index dcd5d91491..648aad3ae1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ classifiers = [ "Topic :: Scientific/Engineering :: Artificial Intelligence", ] dependencies = [ - "unsloth_zoo>=2026.6.3", + "unsloth_zoo>=2026.6.4", "wheel>=0.42.0", "packaging", "torch>=2.4.0,<2.11.0", @@ -91,7 +91,7 @@ triton = [ "triton-windows ; (sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] huggingface = [ - "unsloth_zoo>=2026.6.3", + "unsloth_zoo>=2026.6.4", "wheel>=0.42.0", "packaging", "torchvision", @@ -488,7 +488,7 @@ colab-ampere-torch220 = [ "unsloth[flashattention]", ] colab-new = [ - "unsloth_zoo>=2026.6.3", + "unsloth_zoo>=2026.6.4", "packaging", "tyro", "transformers>=4.51.3,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1,!=4.57.0,!=4.57.4,!=4.57.5,!=5.0.0,!=5.1.0,<=5.5.0", From c9aa094c91e51ab3e8695d14cf26d2df46cee23f Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 13 Jun 2026 04:43:58 -0700 Subject: [PATCH 025/132] Update pyproject.toml --- pyproject.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 648aad3ae1..178dc06d66 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ classifiers = [ "Topic :: Scientific/Engineering :: Artificial Intelligence", ] dependencies = [ - "unsloth_zoo>=2026.6.4", + "unsloth_zoo>=2026.6.5", "wheel>=0.42.0", "packaging", "torch>=2.4.0,<2.11.0", @@ -91,7 +91,7 @@ triton = [ "triton-windows ; (sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] huggingface = [ - "unsloth_zoo>=2026.6.4", + "unsloth_zoo>=2026.6.5", "wheel>=0.42.0", "packaging", "torchvision", @@ -488,7 +488,7 @@ colab-ampere-torch220 = [ "unsloth[flashattention]", ] colab-new = [ - "unsloth_zoo>=2026.6.4", + "unsloth_zoo>=2026.6.5", "packaging", "tyro", "transformers>=4.51.3,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1,!=4.57.0,!=4.57.4,!=4.57.5,!=5.0.0,!=5.1.0,<=5.5.0", From 287243a8fdcceb3d61878fadc5f6ac2282546c4c Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 18 Jun 2026 10:37:48 -0700 Subject: [PATCH 026/132] Update pyproject.toml --- pyproject.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 178dc06d66..a33def608e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ classifiers = [ "Topic :: Scientific/Engineering :: Artificial Intelligence", ] dependencies = [ - "unsloth_zoo>=2026.6.5", + "unsloth_zoo>=2026.6.6", "wheel>=0.42.0", "packaging", "torch>=2.4.0,<2.11.0", @@ -91,7 +91,7 @@ triton = [ "triton-windows ; (sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] huggingface = [ - "unsloth_zoo>=2026.6.5", + "unsloth_zoo>=2026.6.6", "wheel>=0.42.0", "packaging", "torchvision", @@ -488,7 +488,7 @@ colab-ampere-torch220 = [ "unsloth[flashattention]", ] colab-new = [ - "unsloth_zoo>=2026.6.5", + "unsloth_zoo>=2026.6.6", "packaging", "tyro", "transformers>=4.51.3,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1,!=4.57.0,!=4.57.4,!=4.57.5,!=5.0.0,!=5.1.0,<=5.5.0", From 12f34025dc4ba2c55b87cf79925f480868e8ec68 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 22 Jun 2026 09:01:36 -0700 Subject: [PATCH 027/132] Update pyproject.toml --- pyproject.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a33def608e..98f94965ae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ classifiers = [ "Topic :: Scientific/Engineering :: Artificial Intelligence", ] dependencies = [ - "unsloth_zoo>=2026.6.6", + "unsloth_zoo>=2026.6.7", "wheel>=0.42.0", "packaging", "torch>=2.4.0,<2.11.0", @@ -91,7 +91,7 @@ triton = [ "triton-windows ; (sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] huggingface = [ - "unsloth_zoo>=2026.6.6", + "unsloth_zoo>=2026.6.7", "wheel>=0.42.0", "packaging", "torchvision", @@ -488,7 +488,7 @@ colab-ampere-torch220 = [ "unsloth[flashattention]", ] colab-new = [ - "unsloth_zoo>=2026.6.6", + "unsloth_zoo>=2026.6.7", "packaging", "tyro", "transformers>=4.51.3,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1,!=4.57.0,!=4.57.4,!=4.57.5,!=5.0.0,!=5.1.0,<=5.5.0", From 2f111da0f6066214e0c6194a307a188452102bfc Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 7 Jul 2026 06:20:52 -0700 Subject: [PATCH 028/132] Update pyproject.toml --- pyproject.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 98f94965ae..e25da828d4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ classifiers = [ "Topic :: Scientific/Engineering :: Artificial Intelligence", ] dependencies = [ - "unsloth_zoo>=2026.6.7", + "unsloth_zoo>=2026.7.1", "wheel>=0.42.0", "packaging", "torch>=2.4.0,<2.11.0", @@ -91,7 +91,7 @@ triton = [ "triton-windows ; (sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] huggingface = [ - "unsloth_zoo>=2026.6.7", + "unsloth_zoo>=2026.7.1", "wheel>=0.42.0", "packaging", "torchvision", @@ -488,7 +488,7 @@ colab-ampere-torch220 = [ "unsloth[flashattention]", ] colab-new = [ - "unsloth_zoo>=2026.6.7", + "unsloth_zoo>=2026.7.1", "packaging", "tyro", "transformers>=4.51.3,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1,!=4.57.0,!=4.57.4,!=4.57.5,!=5.0.0,!=5.1.0,<=5.5.0", From 3f55e7a98c642b0284e80110e789f52f4e169301 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 8 Jul 2026 07:20:22 -0700 Subject: [PATCH 029/132] Update pyproject.toml --- pyproject.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e25da828d4..29a3c9fe3c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ classifiers = [ "Topic :: Scientific/Engineering :: Artificial Intelligence", ] dependencies = [ - "unsloth_zoo>=2026.7.1", + "unsloth_zoo>=2026.7.2", "wheel>=0.42.0", "packaging", "torch>=2.4.0,<2.11.0", @@ -91,7 +91,7 @@ triton = [ "triton-windows ; (sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] huggingface = [ - "unsloth_zoo>=2026.7.1", + "unsloth_zoo>=2026.7.2", "wheel>=0.42.0", "packaging", "torchvision", @@ -488,7 +488,7 @@ colab-ampere-torch220 = [ "unsloth[flashattention]", ] colab-new = [ - "unsloth_zoo>=2026.7.1", + "unsloth_zoo>=2026.7.2", "packaging", "tyro", "transformers>=4.51.3,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1,!=4.57.0,!=4.57.4,!=4.57.5,!=5.0.0,!=5.1.0,<=5.5.0", From 7beecd279e87dbd5bb5a56d58e57c9e7f01588d5 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 15 Jul 2026 11:43:23 -0700 Subject: [PATCH 030/132] Update pyproject.toml --- pyproject.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 29a3c9fe3c..f8003ec2cf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ classifiers = [ "Topic :: Scientific/Engineering :: Artificial Intelligence", ] dependencies = [ - "unsloth_zoo>=2026.7.2", + "unsloth_zoo>=2026.7.3", "wheel>=0.42.0", "packaging", "torch>=2.4.0,<2.11.0", @@ -91,7 +91,7 @@ triton = [ "triton-windows ; (sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] huggingface = [ - "unsloth_zoo>=2026.7.2", + "unsloth_zoo>=2026.7.3", "wheel>=0.42.0", "packaging", "torchvision", @@ -488,7 +488,7 @@ colab-ampere-torch220 = [ "unsloth[flashattention]", ] colab-new = [ - "unsloth_zoo>=2026.7.2", + "unsloth_zoo>=2026.7.3", "packaging", "tyro", "transformers>=4.51.3,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1,!=4.57.0,!=4.57.4,!=4.57.5,!=5.0.0,!=5.1.0,<=5.5.0", From 6c3c01c0c9580ea284cbf4f5b9e3c4f1f0b07631 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 20 Jul 2026 07:08:29 -0700 Subject: [PATCH 031/132] Update pyproject.toml --- pyproject.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f8003ec2cf..70a5d77a06 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ classifiers = [ "Topic :: Scientific/Engineering :: Artificial Intelligence", ] dependencies = [ - "unsloth_zoo>=2026.7.3", + "unsloth_zoo>=2026.7.4", "wheel>=0.42.0", "packaging", "torch>=2.4.0,<2.11.0", @@ -91,7 +91,7 @@ triton = [ "triton-windows ; (sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] huggingface = [ - "unsloth_zoo>=2026.7.3", + "unsloth_zoo>=2026.7.4", "wheel>=0.42.0", "packaging", "torchvision", @@ -488,7 +488,7 @@ colab-ampere-torch220 = [ "unsloth[flashattention]", ] colab-new = [ - "unsloth_zoo>=2026.7.3", + "unsloth_zoo>=2026.7.4", "packaging", "tyro", "transformers>=4.51.3,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1,!=4.57.0,!=4.57.4,!=4.57.5,!=5.0.0,!=5.1.0,<=5.5.0", From b22b243e1b841ae3a2397739463140a1cb2d7d14 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 23 Jul 2026 03:07:15 -0700 Subject: [PATCH 032/132] Update pyproject.toml --- pyproject.toml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 70a5d77a06..8dc8aa56d4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,10 +25,10 @@ classifiers = [ "Topic :: Scientific/Engineering :: Artificial Intelligence", ] dependencies = [ - "unsloth_zoo>=2026.7.4", + "unsloth_zoo>=2026.7.6", "wheel>=0.42.0", "packaging", - "torch>=2.4.0,<2.11.0", + "torch>=2.4.0,<2.12.0", "torchvision", "numpy", "tqdm", @@ -48,7 +48,7 @@ dependencies = [ "diffusers", "transformers>=4.51.3,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1,!=4.57.0,!=4.57.4,!=4.57.5,!=5.0.0,!=5.1.0,<=5.5.0", "trl>=0.18.2,!=0.19.0,<=0.24.0", - "typer", + "typer>=0.12.0", "pydantic", "pyyaml", "nest-asyncio", @@ -91,7 +91,7 @@ triton = [ "triton-windows ; (sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", ] huggingface = [ - "unsloth_zoo>=2026.7.4", + "unsloth_zoo>=2026.7.6", "wheel>=0.42.0", "packaging", "torchvision", @@ -110,7 +110,7 @@ huggingface = [ "transformers>=4.51.3,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1,!=4.57.0,!=4.57.4,!=4.57.5,!=5.0.0,!=5.1.0,<=5.5.0", "trl>=0.18.2,!=0.19.0,<=0.24.0", "sentence-transformers", - "typer", + "typer>=0.12.0", "pydantic", "pyyaml", "nest-asyncio", @@ -488,7 +488,7 @@ colab-ampere-torch220 = [ "unsloth[flashattention]", ] colab-new = [ - "unsloth_zoo>=2026.7.4", + "unsloth_zoo>=2026.7.6", "packaging", "tyro", "transformers>=4.51.3,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1,!=4.57.0,!=4.57.4,!=4.57.5,!=5.0.0,!=5.1.0,<=5.5.0", @@ -504,7 +504,7 @@ colab-new = [ "bitsandbytes>=0.45.5,!=0.46.0,!=0.48.0", "unsloth[triton]", "sentence-transformers", - "typer", + "typer>=0.12.0", "pydantic", "pyyaml", "nest-asyncio", From 3455b45977bf7c64f1345b04c9c13efd429dab93 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 27 Jul 2026 10:25:55 +0000 Subject: [PATCH 033/132] pip: track main's packaging and tooling config The merge keeps this branch's own dependencies and extras, which are deliberately reduced, but the rest of pyproject.toml should not diverge. Missing here and present on main: - unsloth_cli package-data, node_prebuilt_pins.json and the backend/assets jinja/html globs, so the wheel was not shipping files the code loads - ruff line-length and pytest pythonpath test_pins_manifest_is_declared_in_package_data now passes on this branch. --- pyproject.toml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 8dc8aa56d4..134ceef3a8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,10 +64,12 @@ version = {attr = "unsloth.models._utils.__version__"} include-package-data = true [tool.setuptools.package-data] +unsloth_cli = ["codex_fallback_prompt.md", "pi_subagent.ts"] studio = [ "*.sh", "*.ps1", "*.bat", + "node_prebuilt_pins.json", "frontend/dist/**/*", "frontend/*.json", "frontend/*.ts", @@ -77,6 +79,8 @@ studio = [ "frontend/.git*", "backend/requirements/**/*", "backend/plugins/**/*", + "backend/assets/**/*.jinja", + "backend/assets/**/*.html", "backend/core/data_recipe/oxc-validator/*.json", "backend/core/data_recipe/oxc-validator/*.mjs", ] @@ -756,6 +760,7 @@ repository = "https://github.com/unslothai/unsloth" [tool.ruff] target-version = "py311" +line-length = 100 force-exclude = true extend-exclude = [ "*chat_templates.py", @@ -787,4 +792,5 @@ ignore = [ # Narrow the default test discovery so `pytest` from the repo root # does NOT pick up the GPU-heavy tests under tests/python, tests/qlora, # etc. The CI security job runs `pytest tests/security` explicitly. +pythonpath = ["."] testpaths = ["tests/security"] From 1915ca98dbe6d5a0b6dce5c85cbeb77706a63610 Mon Sep 17 00:00:00 2001 From: Nilay <118994073+NilayYadav@users.noreply.github.com> Date: Mon, 27 Jul 2026 16:08:30 +0530 Subject: [PATCH 034/132] Studio: fetch bare hostnames as https instead of refusing them (#7427) * fetch bare hostnames as https instead of refusing them * normalize host:port URLs and route schemeless github repos to the readme API * only rewrite dotted host:port URLs with in-range ports * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * reject relative paths and oversized ports in url normalization * Match web-fetch ports as ASCII digits so a unicode digit cannot raise str.isdigit() is True for digit-class characters int() refuses (superscript two, circled digit one), so _normalize_url_scheme reached int(port) and raised ValueError out of _fetch_url_raw, which runs before its try block. A web_search url of "example.com:" surfaced a generic tool exception instead of the Blocked: message it returned before this branch. Match the port against an anchored [0-9]{1,5} instead; the five-digit cap that kept the range check from converting an unbounded integer is now in the pattern. * Apply the invalid-port guard to redirect targets too _fetch_url_raw wraps the initial parsed.port in try/except ValueError, but the redirect hop reads rp.port unguarded, so a server answering Location: https://example.org:99999/next fell through to the broad handler as "Failed to fetch URL: Port out of range 0-65535" rather than a deliberate block. No request is dispatched either way; this just makes the two paths report the same way. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Keep the redirect-port test compact The formatter expands a signature carrying a spaced kwarg default, which put the stub opener on eleven lines. **kw absorbs the timeout the fetch loop passes and leaves the whole stub on four. * Never let a malformed URL escape _fetch_url_raw as an exception The URL is model-supplied, so every bad form should come back as one of the documented (error, body, content_type) strings. Three gaps remained: urlparse itself raises on an unmatched IPv6 bracket and on a netloc that NFKC-decomposes into a delimiter (//exam(fullwidth-solidus)ple.com), and both calls sat outside a guard. getaddrinfo raises UnicodeError, which is a ValueError and not the OSError _validate_and_resolve_host catches, when IDNA encoding rejects a hostname. Over a 3158 URL corpus that injects tabs, newlines, C0 controls, delimiters and NFKC confusables at every position, main raises 42 times and this raises none. Also strip surrounding whitespace in _normalize_url_scheme. _web_search already stripped, but normalization moved down to the fetch layer, so a direct _fetch_page_text caller did not get it. * Name the host in the status badge and tool card for bare URLs status_for_tool and the web-search tool card both required an explicit scheme before reading the hostname, so every URL this branch newly makes fetchable showed the generic "Reading page..." and "Read page" instead of the host. Under permission_mode=ask that means the approval card named no destination for exactly the inputs the branch enables. The backend reuses _normalize_url_scheme. The frontend cannot, since new URL() throws on a bare host, so RE_BARE_HOST mirrors the same grammar: only a dotted host with an optional in-range port gets the https prefix, leaving /login, javascript: and userinfo forms to render generically as before. Also mention bare hostnames in the url parameter description, since they are part of the accepted interface now. * Do not let a malformed URL in the status badge kill the tool turn status_for_tool runs inside prepare_call, before the fetch and outside the handler that wraps tool execution, so a ValueError from urlparse ends the whole turn instead of letting _fetch_url_raw return its blocked message. _normalize_url_scheme catches its own parse error and hands back the original string, so the parse here still has to be guarded. Reachable with https://[::1 or a host that NFKC-decomposes into a delimiter. This predates the branch, main raises identically, but the badge is one of the lines this branch touches and the rest of it already promises no malformed URL escapes as an exception. * Tighten the comments added by this branch * Revert the web_search url description change The premise of this branch is that models already emit bare hostnames unprompted, which is why the fetch layer had to stop refusing them. Advertising the bare form in the tool schema does not enable anything, it just steers models toward it, and that is the form carrying every edge case: ambiguous with dotted custom schemes, and unlike an explicit scheme it does not cover IPv6 literals, IDN or trailing-dot FQDNs. The fetch layer tolerates bare hosts. The schema should keep recommending a full URL. This also drops the one change here with no regression test. * Match the backend port rule in the tool card host The card's bare-host pattern required at least one digit after the colon, but the backend fetches an empty port (example.com: and example.com:/path go to the default HTTPS port), so a successful fetch rendered as "Read page" with no host. Allowing an empty port alone would have swung it the other way: example.com:0 is refused by the backend but new URL() accepts it, so the card would have named a host that is never fetched. That mismatch was there before this change too. Mirror the backend rule instead, an empty port or one in 1-65535, checked against every case in the normalizer's own matrix. --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Daniel Han --- .../core/inference/tool_loop_controller.py | 11 +- studio/backend/core/inference/tools.py | 67 ++++++- .../tests/test_tool_loop_controller.py | 25 +++ .../test_web_fetch_scheme_normalization.py | 170 ++++++++++++++++++ .../assistant-ui/tool-ui-web-search.tsx | 18 +- 5 files changed, 286 insertions(+), 5 deletions(-) create mode 100644 studio/backend/tests/test_web_fetch_scheme_normalization.py diff --git a/studio/backend/core/inference/tool_loop_controller.py b/studio/backend/core/inference/tool_loop_controller.py index 61643b5795..361f4b20e3 100644 --- a/studio/backend/core/inference/tool_loop_controller.py +++ b/studio/backend/core/inference/tool_loop_controller.py @@ -212,7 +212,16 @@ def status_for_tool(tool_name: str, arguments: Mapping[str, Any]) -> str: if tool_name == "web_search": url = str(arguments.get("url") or "").strip() if url: - parsed = urlparse(url) + # Bare hosts are fetched as https, so normalize first or the badge + # stays generic for exactly the URLs the fetch layer accepts. + from core.inference.tools import _normalize_url_scheme + + try: + parsed = urlparse(_normalize_url_scheme(url)) + except ValueError: + # Runs in prepare_call, outside the fetch's exception handler: + # raising here kills the turn instead of returning "Blocked:". + return "Reading page..." if parsed.scheme in ("http", "https") and parsed.hostname: host = parsed.hostname if host.startswith("www."): diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.py index d45fede89a..bd5322819e 100644 --- a/studio/backend/core/inference/tools.py +++ b/studio/backend/core/inference/tools.py @@ -6332,7 +6332,8 @@ def _validate_and_resolve_host(hostname: str, port: int) -> tuple[bool, str, str try: infos = socket.getaddrinfo(hostname, port, type = socket.SOCK_STREAM) - except OSError as e: + except (OSError, UnicodeError) as e: + # IDNA encoding rejects a hostname with UnicodeError, not OSError. return False, f"Failed to resolve host: {e}", "" if not infos: @@ -6562,6 +6563,56 @@ def _read_capped_body(resp, max_bytes, timeout, deadline, cancel_event): return None, b"".join(chunks) +_DOTTED_HOST_RE = re.compile(r"[A-Za-z0-9-]+(\.[A-Za-z0-9-]+)+") +# ASCII-only because str.isdigit() is True for digits int() refuses ("²"), and +# capped at 5 digits so the range check never converts an unbounded integer. +_PORT_RE = re.compile(r"[0-9]{1,5}") + + +def _normalize_url_scheme(url: str) -> str: + """Prepend ``https://`` to bare hosts (``google.com``, ``example.com:8443``). + + ``urlparse`` reads the host of a ``host:port`` input as the scheme, so those + are recognised by a dotted host-like scheme with an empty netloc. Rewrites a + dotted host with an optional in-range port, and the ``//host`` form. Real + schemes (``file:``, ``javascript:``, including ``file:80``), root-relative + paths (``/login``) and bad ports are returned untouched so the caller + rejects them. A dotted scheme is indistinguishable from ``host:port``, so + ``com.acme.app:443/cb`` is rewritten too; an empty port (``example.com:``) + is kept as-is, matching ``https://example.com:``. + + The host is matched against the raw authority, never against what + ``urlparse`` returned, because urlsplit strips tabs/newlines (3.10) and + leading C0/space (3.12). Anything it would strip fails the match, so the + decision and the rewritten string cannot disagree across versions.""" + from urllib.parse import urlparse + + url = url.strip() + try: + parsed = urlparse(url) + except ValueError: + # Unmatched IPv6 brackets, or an NFKC-decomposing netloc: not a bare host. + return url + if parsed.scheme: + if parsed.netloc or not _DOTTED_HOST_RE.fullmatch(parsed.scheme): + return url + rest = url + elif url.startswith("//"): + rest = url[2:] + elif url.startswith("/"): + return url + else: + rest = url + + authority = re.split(r"[/?#]", rest, maxsplit = 1)[0] + host, _, port = authority.partition(":") + if not _DOTTED_HOST_RE.fullmatch(host): + return url + if port and not (_PORT_RE.fullmatch(port) and 1 <= int(port) <= 65535): + return url + return "https://" + rest + + def _fetch_url_raw( url: str, timeout: int = 30, @@ -6575,6 +6626,8 @@ def _fetch_url_raw( ``error`` is a user-facing message string when the fetch failed (the existing "Blocked:" / "Failed to fetch URL:" wording), else ``None``. Blocks private/loopback/link-local targets and caps the download size. + No input reaches the caller as an exception: the URL is model-supplied, so + every malformed form resolves to one of these strings. ``deadline`` is an optional ``time.monotonic`` cutoff for the whole fetch (redirect hops and body read included) and ``cancel_event`` aborts it when @@ -6583,11 +6636,15 @@ def _fetch_url_raw( from urllib.parse import urlparse from .web_access_policy import check_url_access - parsed = urlparse(url) + # Before the policy gate: it requires an http(s) scheme, so a bare host + # would be refused there and never reach the fetch. + url = _normalize_url_scheme(url) allowed, reason, canonical_host = check_url_access(url, website_policy) if not allowed: return reason, "", "" + # check_url_access already parsed this and read .port, so this cannot raise. + parsed = urlparse(url) port = parsed.port or (443 if parsed.scheme == "https" else 80) ok, reason, pinned_ip = _resolve_with_budget( canonical_host, @@ -6648,13 +6705,15 @@ def _fetch_url_raw( if not location: return "Failed to fetch URL: redirect missing Location header.", "", "" current_url = urljoin(current_url, location) - rp = urlparse(current_url) + # Server-controlled, so never scheme-upgraded; the gate below + # reads .port first, so the parse after it cannot raise. allowed, policy_reason, redirect_host = check_url_access( current_url, website_policy, ) if not allowed: return policy_reason, "", "" + rp = urlparse(current_url) rp_port = rp.port or (443 if rp.scheme == "https" else 80) ok2, reason2, pinned_ip = _resolve_with_budget( redirect_host, @@ -6872,6 +6931,8 @@ def _fetch_page_text( deadline = None if timeout is None else time.monotonic() + timeout from .web_access_policy import check_url_access + # Before the policy gate (needs a scheme) and the README routing (reads host/path). + url = _normalize_url_scheme(url) allowed, reason, _hostname = check_url_access(url, website_policy) if not allowed: return reason diff --git a/studio/backend/tests/test_tool_loop_controller.py b/studio/backend/tests/test_tool_loop_controller.py index 496c30ac13..e9ed58b090 100644 --- a/studio/backend/tests/test_tool_loop_controller.py +++ b/studio/backend/tests/test_tool_loop_controller.py @@ -7,6 +7,8 @@ import json import sys from pathlib import Path +import pytest + _BACKEND_DIR = str(Path(__file__).resolve().parent.parent) if _BACKEND_DIR not in sys.path: sys.path.insert(0, _BACKEND_DIR) @@ -93,6 +95,29 @@ def test_status_and_provenance_match_local_event_conventions(): } +@pytest.mark.parametrize( + "url, expected", + [ + # bare hosts are fetched, so the badge must name them + ("google.com", "Reading: google.com"), + ("www.google.com/x", "Reading: google.com"), + ("//google.com", "Reading: google.com"), + ("example.com:8443/path", "Reading: example.com"), + ("github.com/unslothai/unsloth", "Reading: github.com"), + # still generic for what the fetch layer refuses + ("/login", "Reading page..."), + ("javascript:alert(1)", "Reading page..."), + # urlparse raises on these, outside the fetch's handler: degrade, not raise + ("https://[::1", "Reading page..."), + ("https://::1]", "Reading page..."), + ("//exam／ple.com", "Reading page..."), + ("//example.com＠", "Reading page..."), + ], +) +def test_status_names_the_host_for_schemeless_urls(url, expected): + assert status_for_tool("web_search", {"url": url}) == expected + + def test_prepare_execute_builds_visible_events_and_model_tool_message(): controller = ToolLoopController(tools = [_tool("web_search")]) decision = controller.prepare_call(_call("web_search", {"query": "gpu prices"})) diff --git a/studio/backend/tests/test_web_fetch_scheme_normalization.py b/studio/backend/tests/test_web_fetch_scheme_normalization.py new file mode 100644 index 0000000000..b4dad837e6 --- /dev/null +++ b/studio/backend/tests/test_web_fetch_scheme_normalization.py @@ -0,0 +1,170 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Bare hosts ("google.com") must be fetched as https, not refused.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + +_BACKEND = Path(__file__).resolve().parent.parent +if str(_BACKEND) not in sys.path: + sys.path.insert(0, str(_BACKEND)) + +from core.inference import tools # noqa: E402 + + +@pytest.fixture +def resolved(monkeypatch): + seen: dict = {} + + def fake_resolve(hostname, port, deadline, cancel_event): + seen["hostname"] = hostname + seen["port"] = port + return False, "stopped", None + + monkeypatch.setattr(tools, "_resolve_with_budget", fake_resolve) + return seen + + +@pytest.mark.parametrize( + "url, hostname, port", + [ + ("google.com", "google.com", 443), + ("www.google.com/x", "www.google.com", 443), + ("//google.com", "google.com", 443), + ("https://google.com", "google.com", 443), + ("http://google.com", "google.com", 80), + ("example.com:8443/path", "example.com", 8443), + ("example.com:8443", "example.com", 8443), + ("sub.example.co.uk:8080", "sub.example.co.uk", 8080), + ], +) +def test_schemeless_urls_are_fetched_as_https(resolved, url, hostname, port): + err, _, _ = tools._fetch_url_raw(url) + assert resolved["hostname"] == hostname + assert resolved["port"] == port + assert "only http/https" not in (err or "") + + +@pytest.mark.parametrize( + "url", + [ + "ftp://x.com", + "file:///etc/passwd", + "javascript:alert(1)", + "mailto:a@b.c", + # scheme:digits must not masquerade as host:port + "file:80", + "javascript:443/path", + "mailto:25", + # out-of-range ports are not host:port either + "example.com:99999", + "example.com:0", + # ports must match ASCII [0-9]: str.isdigit() is True for digits int() refuses + "example.com:²", + "example.com:²/x", + "example.com:①", + "example.com:1²", + "//example.com:²", + # non-ASCII decimal digits int() accepts are ports urlparse then refuses + "example.com:٤٤٣", + # root-relative paths have no host to fetch + "/login", + "/github.com/owner/repo", + ], +) +def test_non_http_schemes_still_blocked(url): + err, _, _ = tools._fetch_url_raw(url) + assert err and "only http/https" in err + + +def test_absurdly_long_port_does_not_raise(): + err, _, _ = tools._fetch_url_raw("example.com:" + "9" * 4400) + assert err and "only http/https" in err + + +def test_out_of_range_port_returns_error_instead_of_raising(): + # check_url_access owns the wording; what matters is a string, not a raise. + err, _, _ = tools._fetch_url_raw("https://example.com:99999") + assert err and err.startswith("Blocked:") + + +def test_redirect_to_out_of_range_port_is_blocked(monkeypatch): + # A redirect target reads .port too, so it needs the same guard. + import urllib.request + from urllib.error import HTTPError + + monkeypatch.setattr( + tools, + "_resolve_with_budget", + lambda host, port, deadline, cancel: (True, "", "93.184.216.34"), + ) + + class _Redirecting: + def open(self, req, **kw): + hdrs = {"Location": "https://example.org:99999/next"} + raise HTTPError(req.full_url, 302, "Found", hdrs, None) + + monkeypatch.setattr(urllib.request, "build_opener", lambda *handlers: _Redirecting()) + err, _, _ = tools._fetch_url_raw("https://example.com") + assert err and err.startswith("Blocked:") + + +@pytest.mark.parametrize( + "url", + [ + # urlparse raises on these; a model-supplied URL must still return a string + "//exam／ple.com", # NFKC-decomposes into "/" + "//example.com＠", # NFKC-decomposes into "@" + "//example.com：", # NFKC-decomposes into ":" + "https://[::1", # unmatched IPv6 bracket + "https://::1]", + ], +) +def test_malformed_url_is_blocked_instead_of_raising(url): + err, _, _ = tools._fetch_url_raw(url) + assert err and err.startswith("Blocked:") + + +def test_idna_failure_is_reported_instead_of_raising(monkeypatch): + # getaddrinfo raises UnicodeError, not OSError, when IDNA encoding fails. + import socket + + def boom(*a, **k): + raise UnicodeError("encoding with 'idna' codec failed") + + monkeypatch.setattr(socket, "getaddrinfo", boom) + err, _, _ = tools._fetch_url_raw("https://münich.example") + assert err and err.startswith("Failed to resolve host:") + + +@pytest.mark.parametrize( + "url, hostname", + [ + (" google.com", "google.com"), + ("google.com\n", "google.com"), + ("\t example.com:8443 ", "example.com"), + ], +) +def test_surrounding_whitespace_is_stripped(resolved, url, hostname): + # _web_search strips, but direct callers of the fetch layer do not. + tools._fetch_url_raw(url) + assert resolved["hostname"] == hostname + + +@pytest.mark.parametrize("url", ["127.0.0.1", "169.254.169.254", "10.0.0.1", "192.168.1.1"]) +def test_normalization_does_not_bypass_ssrf_guard(url): + err, _, _ = tools._fetch_url_raw(url, timeout = 3) + assert err and "non-public address" in err + + +def test_schemeless_github_repo_still_routes_to_readme_api(): + # Must run before _github_repo_readme_api_url, else a bare repo URL scrapes HTML. + normalized = tools._normalize_url_scheme("github.com/unslothai/unsloth") + assert tools._github_repo_readme_api_url(normalized) == ( + "https://api.github.com/repos/unslothai/unsloth/readme" + ) diff --git a/studio/frontend/src/components/assistant-ui/tool-ui-web-search.tsx b/studio/frontend/src/components/assistant-ui/tool-ui-web-search.tsx index 062d0b1370..e11ef6cc2a 100644 --- a/studio/frontend/src/components/assistant-ui/tool-ui-web-search.tsx +++ b/studio/frontend/src/components/assistant-ui/tool-ui-web-search.tsx @@ -23,6 +23,18 @@ const RE_BLOCK_SEP = /\n---\n/; const RE_TITLE = /Title:\s*(.+)/; const RE_URL = /URL:\s*(.+)/; const RE_SNIPPET = /Snippet:\s*(.+)/s; +// Mirrors _normalize_url_scheme: a dotted host, optionally followed by a port +// that may be empty ("example.com:" fetches on the default port) but otherwise +// has to be in range, so the card names a host only when the backend fetches it. +const RE_BARE_HOST = /^[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)+(?::(\d{0,5}))?(?:[/?#]|$)/; + +function isBareHostFetchedAsHttps(value: string): boolean { + const match = RE_BARE_HOST.exec(value); + if (!match) return false; + const port = match[1]; + if (!port) return true; + return Number(port) >= 1 && Number(port) <= 65535; +} /** * Reject non-http(s) URLs. Web-search/fetch output is provider-controlled, @@ -72,8 +84,12 @@ const WebSearchToolUIImpl: ToolCallMessagePartComponent = ({ const isUrlFetch = !!url; const displayDomain = (() => { if (!url) return ""; + // new URL() throws on the bare hosts the backend fetches, so mirror that + // grammar or the card names no host for exactly the URLs it does fetch. + const bare = url.startsWith("//") ? url.slice(2) : url; + const candidate = isBareHostFetchedAsHttps(bare) ? `https://${bare}` : url; try { - const parsed = new URL(url); + const parsed = new URL(candidate); if (parsed.protocol !== "http:" && parsed.protocol !== "https:") return ""; return parsed.hostname.replace(/^www\./, ""); } catch { From 735fcde44cdb6522d1c636fc07b3d3b99be0112d Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 27 Jul 2026 03:44:07 -0700 Subject: [PATCH 035/132] pip: declare the Studio dependencies the wheel's own modules import (#7493) * pip: declare the Studio dependencies the wheel's own modules import The wheel packages studio/ and studio.backend*, so pip install unsloth puts five commands on PATH -- train, export, chat, inference, studio -- and every one of them imports studio.backend.*. None of those imports were declared, so all five ended in a rich traceback at ModuleNotFoundError: No module named 'structlog' (#4701, #5260, #7147). --help rendered fine for all of them because typer defers the import, which is why this went unnoticed. Walking module-level, non-try-guarded imports from each entry point shows structlog is the only hard requirement they share, once starlette's annotation-only import in loggers/handlers.py moves under TYPE_CHECKING. So structlog becomes a core dependency and the rest of the server stack (fastapi, uvicorn, matplotlib, pandas, pymupdf, ...) becomes a [studio] extra mirroring studio/backend/requirements/studio.txt, with a test that fails if the two drift. pip install unsloth -> train / export work pip install "unsloth[studio]" -> the server works * Apply ruff-format quote normalisation * Trim the comments added in this PR --- pyproject.toml | 30 +++++++ studio/backend/loggers/handlers.py | 9 +- .../test_studio_extra_matches_requirements.py | 84 +++++++++++++++++++ 3 files changed, 122 insertions(+), 1 deletion(-) create mode 100644 tests/studio/install/test_studio_extra_matches_requirements.py diff --git a/pyproject.toml b/pyproject.toml index 134ceef3a8..fdbee69c58 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,6 +52,9 @@ dependencies = [ "pydantic", "pyyaml", "nest-asyncio", + # Every CLI command imports studio.backend.*, which reaches structlog at + # module level. The rest of the server stack lives in the studio extra. + "structlog>=24.1.0", ] [project.scripts] @@ -90,6 +93,33 @@ include = ["unsloth*", "unsloth_cli*", "studio", "studio.backend*"] exclude = ["images*", "tests*", "*.node_modules", "*.node_modules.*"] [project.optional-dependencies] +# Studio's server stack. Mirrors studio/backend/requirements/studio.txt; +# test_studio_extra_matches_requirements.py catches drift. +studio = [ + "typer", + "fastapi", + "uvicorn", + "pydantic", + "packaging", + "matplotlib==3.10.9", + "pandas", + "nest_asyncio", + "datasets==4.3.0", + "pyjwt", + "huggingface-hub==0.36.2", + "structlog>=24.1.0", + "diceware", + "ddgs", + "cryptography>=42.0.0", + "boto3>=1.34.0", + "httpx>=0.27.0", + "fastmcp>=3.0.2", + "sqlite-vec==0.1.9", + "pymupdf==1.27.2.3", + "pymupdf4llm==0.3.4", + "python-docx==1.2.0", +] + triton = [ "triton>=3.0.0 ; ('linux' in sys_platform)", "triton-windows ; (sys_platform == 'win32') and (platform_machine == 'AMD64' or platform_machine == 'x86_64')", diff --git a/studio/backend/loggers/handlers.py b/studio/backend/loggers/handlers.py index 716c4f40d2..7f085439d4 100644 --- a/studio/backend/loggers/handlers.py +++ b/studio/backend/loggers/handlers.py @@ -8,12 +8,19 @@ filter_sensitive_data (structlog processor for sanitization), and get_logger (factory for structured loggers). """ +from __future__ import annotations + import os import re import time +from typing import TYPE_CHECKING import structlog -from starlette.types import ASGIApp, Message, Receive, Scope, Send + +# Annotations only: importing at runtime would make the ASGI stack a hard +# dependency of every CLI command. +if TYPE_CHECKING: + from starlette.types import ASGIApp, Message, Receive, Scope, Send from utils.native_path_leases import redact_native_paths diff --git a/tests/studio/install/test_studio_extra_matches_requirements.py b/tests/studio/install/test_studio_extra_matches_requirements.py new file mode 100644 index 0000000000..a74d52ff40 --- /dev/null +++ b/tests/studio/install/test_studio_extra_matches_requirements.py @@ -0,0 +1,84 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""The studio extra must mirror studio/backend/requirements/studio.txt. + +Nothing else keeps them in sync, and drift reintroduces #4701 / #5260 / #7147. +""" + +from __future__ import annotations + +import pathlib +import sys + +import pytest + +REPO_ROOT = pathlib.Path(__file__).resolve().parents[3] +PYPROJECT = REPO_ROOT / "pyproject.toml" +STUDIO_TXT = REPO_ROOT / "studio" / "backend" / "requirements" / "studio.txt" + +# Imported at module scope by the studio.backend chain every CLI command walks. +CORE_RUNTIME_PACKAGES = ("structlog",) + + +def _load_pyproject() -> dict: + if sys.version_info >= (3, 11): + import tomllib + else: + tomllib = pytest.importorskip("tomli") + return tomllib.loads(PYPROJECT.read_text(encoding = "utf-8")) + + +def _requirement_lines(path: pathlib.Path) -> list[str]: + out = [] + for line in path.read_text(encoding = "utf-8").splitlines(): + text = line.split("#", 1)[0].strip() + if text and not text.startswith("-"): + out.append(text) + return out + + +def _normalise(name: str) -> str: + """PEP 503 normalisation, so PyJWT/pyjwt and nest_asyncio/nest-asyncio match.""" + head = name + for sep in ("===", "==", ">=", "<=", "~=", "!=", ">", "<", "[", ";", " "): + idx = head.find(sep) + if idx > 0: + head = head[:idx] + return head.strip().lower().replace("_", "-").replace(".", "-") + + +def test_studio_extra_exists(): + extras = _load_pyproject()["project"]["optional-dependencies"] + assert "studio" in extras, ( + "pyproject.toml has no `studio` extra. The wheel ships studio/ and " + "studio.backend*, so their dependencies need a pip-installable home." + ) + + +def test_studio_extra_matches_requirements_file(): + extras = _load_pyproject()["project"]["optional-dependencies"] + extra = sorted(_normalise(entry) for entry in extras["studio"]) + required = sorted(_normalise(entry) for entry in _requirement_lines(STUDIO_TXT)) + + missing = sorted(set(required) - set(extra)) + surplus = sorted(set(extra) - set(required)) + assert not missing, ( + f"studio.txt lists {missing} but the `studio` extra does not. " + '`pip install "unsloth[studio]"` would build a venv the Studio server ' + "cannot boot in. Add them to [project.optional-dependencies] studio." + ) + assert not surplus, ( + f"The `studio` extra lists {surplus} but studio.txt does not. " + "Remove them, or add them to studio.txt if install.sh needs them too." + ) + + +@pytest.mark.parametrize("package", CORE_RUNTIME_PACKAGES) +def test_cli_runtime_packages_are_core_dependencies(package): + core = [_normalise(entry) for entry in _load_pyproject()["project"]["dependencies"]] + assert _normalise(package) in core, ( + f"{package} is imported at module scope by the studio.backend chain " + f"`unsloth train` / `unsloth export` walk, so a plain `pip install " + f"unsloth` must provide it or they die with ModuleNotFoundError." + ) From ef97f3c961acf7fe724f95edc61c16c00f5da39c Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 27 Jul 2026 03:46:38 -0700 Subject: [PATCH 036/132] tests: record the gfx1152 llama.cpp bundle gap so the next one is not silent (#7491) * tests: record the gfx1152 llama.cpp bundle gap so the next one is not silent #7431 made gfx1152 (Krackan Point, Radeon 860M/840M) a first-class arch, which fixed torch wheel selection: those laptops were pulling gfx1150 wheels built for a different LLVM target. It also changed llama.cpp prebuilt selection, because no gfx1152 bundle is published. published_rocm_choice_for_host deliberately refuses to serve a sibling-family bundle, so those hosts now fall back to a HIP source build. That is the right outcome, a wrong-ISA binary fails at the first BLAS call rather than merely installing slowly, but nothing recorded it and nothing would have caught it. TestPublishedRocmGfxSelection builds its release from a hardcoded family list, so it can only assert about arches someone already thought to add. Adds TestPublishedRocmBundleCoverage: - PUBLISHED mirrors the mapped_targets in llama-prebuilt-manifest.json. - KNOWN_GAPS lists arches _GFX_TO_AMD_INDEX_ARCH routes torch for that no bundle covers: gfx1033/1035/1036 (RDNA 2, never built) and gfx1152. - test_known_gaps_fall_back_to_source_build pins each to None. - test_every_torch_routed_arch_is_covered_or_a_known_gap compares the routed set against bundle coverage, so adding an arch for torch without a bundle has to be a deliberate KNOWN_GAPS entry. The invariant fires both ways. Simulating a new routed arch fails with "coverage drifted: ['gfx1153'] newly uncovered"; simulating a published gfx1152 bundle fails with "gfx1152 is in KNOWN_GAPS but a bundle now matches it; drop it from the set", so closing the gap cannot leave the list stale. Reads _GFX_TO_AMD_INDEX_ARCH from source instead of importing install_python_stack, which this suite does not otherwise depend on. No production code changes. Install suite 1355 passed, no new failures. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- tests/studio/install/test_selection_logic.py | 94 ++++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/tests/studio/install/test_selection_logic.py b/tests/studio/install/test_selection_logic.py index 8ebcd6cc86..15537bb2bf 100644 --- a/tests/studio/install/test_selection_logic.py +++ b/tests/studio/install/test_selection_logic.py @@ -2935,6 +2935,100 @@ class TestPublishedRocmGfxSelection: assert choice.name == "app-b9457-windows-x64-rocm-gfx120X.zip" +class TestPublishedRocmBundleCoverage: + """Every arch the installer routes torch for should also have a llama.cpp + bundle, or be recorded here as a known gap. Routing an arch for torch while + no bundle covers it is silent: the GPU works for training and drops to a HIP + source build for inference, which is correct but much slower to install.""" + + # mapped_targets of each published ROCm bundle, mirroring + # unslothai/llama.cpp's llama-prebuilt-manifest.json. + PUBLISHED = { + "gfx103X": ["gfx1030", "gfx1031", "gfx1032", "gfx1034"], + "gfx110X": ["gfx1100", "gfx1101", "gfx1102", "gfx1103"], + "gfx120X": ["gfx1200", "gfx1201"], + "gfx1150": ["gfx1150"], + "gfx1151": ["gfx1151"], + "gfx908": ["gfx908"], + "gfx90a": ["gfx90a"], + } + + # Arches _GFX_TO_AMD_INDEX_ARCH routes torch for that no bundle covers. + # gfx1033/1035/1036: RDNA 2 variants, never built. + # gfx1152: Krackan Point (Radeon 860M/840M). Torch goes to its own + # repo.amd.com/rocm/whl/gfx1152 leaf, but no llama.cpp bundle exists, so + # these hosts source-build. Publish a -gfx1152 bundle, or add gfx1152 to + # the gfx1150 bundle's mapped_targets if that build genuinely covers it, + # then drop it from this set. + KNOWN_GAPS = {"gfx1033", "gfx1035", "gfx1036", "gfx1152"} + + def _release(self): + return make_release( + [ + make_artifact( + f"app-b9457-linux-x64-rocm-{fam}.tar.gz", + install_kind = "linux-rocm", + runtime_line = None, + coverage_class = None, + supported_sms = [], + min_sm = None, + max_sm = None, + bundle_profile = None, + rank = 1000, + gfx_target = fam, + mapped_targets = targets, + ) + for fam, targets in self.PUBLISHED.items() + ], + upstream_tag = "b9457", + ) + + def _host(self, gfx): + return make_host( + machine = "x86_64", + nvidia_smi = None, + driver_cuda_version = None, + compute_caps = [], + has_physical_nvidia = False, + has_usable_nvidia = False, + has_rocm = True, + rocm_gfx_target = gfx, + ) + + def test_known_gaps_fall_back_to_source_build(self): + """A gap arch must return None rather than be served a sibling bundle: + a wrong-ISA binary fails at the first BLAS call instead of installing + slowly, which is the worse of the two outcomes.""" + release = self._release() + for gfx in sorted(self.KNOWN_GAPS): + assert ( + INSTALL_LLAMA_PREBUILT.published_rocm_choice_for_host( + release, self._host(gfx), "linux-rocm" + ) + is None + ), f"{gfx} is in KNOWN_GAPS but a bundle now matches it; drop it from the set" + + def test_every_torch_routed_arch_is_covered_or_a_known_gap(self): + """The guard that would have caught gfx1152: adding an arch to + _GFX_TO_AMD_INDEX_ARCH without a bundle must be a deliberate entry in + KNOWN_GAPS, not an unnoticed drop to source builds.""" + import re + + # Read the table from source rather than importing the installer module, + # which pulls in a heavy dependency chain this suite does not need. + stack = (PACKAGE_ROOT / "studio" / "install_python_stack.py").read_text(encoding = "utf-8") + body = re.search(r"_GFX_TO_AMD_INDEX_ARCH.*?=\s*\{(.*?)\n\}", stack, re.S) + assert body, "_GFX_TO_AMD_INDEX_ARCH not found in install_python_stack.py" + routed = set(re.findall(r'"(gfx[0-9a-z]+)":', body.group(1))) + assert routed, "parsed no arches out of _GFX_TO_AMD_INDEX_ARCH" + covered = {t.lower() for targets in self.PUBLISHED.values() for t in targets} + uncovered = {a for a in routed if a.lower() not in covered} + assert uncovered == self.KNOWN_GAPS, ( + f"llama.cpp bundle coverage drifted: {sorted(uncovered - self.KNOWN_GAPS)} " + f"newly uncovered, {sorted(self.KNOWN_GAPS - uncovered)} no longer a gap" + ) + + class TestPublishedMacosForkSelection: """macOS routes to the fork's llama--bin-macos-.tar.gz, selected by install_kind.""" From 7a9749eb4f303d0eb8f8cd05d31fcb8a658d3163 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 27 Jul 2026 04:18:22 -0700 Subject: [PATCH 037/132] unsloth start: keep the local subagent unattended and out of plan mode (#7437) * unsloth start: keep the local subagent unattended and out of plan mode The local subagent child could stall waiting on a permission prompt, and a parent session in plan mode could still reach the editing agent. - Drop human-blocking tools from the child so it runs unattended. The read-only child also drops the file writers. - Emit a PreToolUse hook that reads permission_mode itself and denies the editing agent under plan mode, so routing holds when the model ignores SKILL.md. Fails open, and is skipped under the WSL bridge where a Windows interpreter path is not runnable in the distro. * Make the read-only subagent actually read-only, and drop stale WSL gates From the first review of this branch, which drove the real code against a fake HOME holding a pre-existing Claude install and diffed the tree before and after. No config, agent, MCP server or CLAUDE.md of the user's was touched in either arm, and the session dir is removed on exit, Ctrl-C and exception. Three real findings came out of it: - The read-only child could still write. Plan mode routes Bash through a safety classifier served by the same local model, so a small model saying yes is what authorised the write; a child spawned with read_only created a file. Denying Bash there makes the label true, at the cost of shell exploration while planning. Read, Grep and Glob still cover the search it needs. - A persisted plugin dir kept a plan_gate.py from an earlier Windows run, so a later WSL run shipped a hooks.json naming an interpreter the distro cannot execute. Hook errors do not block, so this only ever wasted a spawn, but it accumulated and the branch had no test. - The comment claimed the read-only child keeps ExitPlanMode "as Claude does under plan mode". A --print child is never offered the plan or prompt tools at all, so most of both deny lists is inert today. Kept as a guard against a version that starts offering them, but the comment now says so. Also covers "auto" in the gate's non-plan modes, which is a real permission_mode and the one the child's own Bash classifier runs under. * Stop the gate failing closed, and bound a wedged child Second review of this branch, driving real claude 2.1.219 against a mock endpoint rather than reading. The gate could fail closed. If plan_gate.py went missing the interpreter exited 2, which Claude treats as a blocking hook error, so the editing tool was denied in every mode rather than just plan. Running the script through runpy instead of handing its path to the interpreter turns that into an ordinary traceback, which is exit 1 and allows. Verified both exit codes directly. The hook also had no timeout, so a hung one stalled the parent for as long as it hung, measured past 400s. Bounded at 10s. The real stall this branch is named for was untouched: run_local_agent polled communicate() forever, so a local server that accepts and never answers left the child and the parent blocked indefinitely, measured past 400s. Added a wall-clock deadline that kills the child and says the server looks wedged. UNSLOTH_CLAUDE_SUBAGENT_TIMEOUT overrides it, 0 restores the old behaviour. Also corrected the plan-mode comment. Claude already refuses the editing tool in plan mode on its own, since it advertises readOnlyHint false; what the hook adds is a reason naming the read-only tool to call instead. The WSL comment had the direction backwards: the gate is the Linux path, not the Windows one. Tests: the hook command's quoting and its behaviour with the gate deleted, both previously unguarded, plus the timeout and its env override. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Keep the gate path out of the shell string Codex review. The hook command is run by a shell, and the gate path was interpolated into it, so a session-config root containing shell metacharacters expanded before Python saw it. Verified on both: sh expands $(..), backticks and $VAR; cmd expands %VAR%. In every case the path no longer resolves, the gate exits 1, and because that intentionally fails open the routing message silently stops appearing. The path now travels as base64, whose alphabet has no metacharacter in either shell. Parametrised over all four hostile forms, and the old interpolation makes those tests fail. One correction to the report: it says the editing agent becomes callable in plan mode. It does not. Claude refuses that tool by itself, since it advertises readOnlyHint false, which was checked earlier by deleting the hook entirely. What a mangled path costs is the reason naming the read-only agent to call instead, not the block. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Lee Jackson <130007945+Imagineer99@users.noreply.github.com> --- unsloth_cli/claude_subagent_mcp.py | 36 ++++ unsloth_cli/commands/start.py | 73 +++++++ unsloth_cli/tests/test_claude_plan_gate.py | 179 ++++++++++++++++++ unsloth_cli/tests/test_claude_subagent_mcp.py | 50 +++++ 4 files changed, 338 insertions(+) create mode 100644 unsloth_cli/tests/test_claude_plan_gate.py diff --git a/unsloth_cli/claude_subagent_mcp.py b/unsloth_cli/claude_subagent_mcp.py index e044d78705..b66d3c7ab9 100644 --- a/unsloth_cli/claude_subagent_mcp.py +++ b/unsloth_cli/claude_subagent_mcp.py @@ -29,6 +29,10 @@ from unsloth_cli.commands.start import ( _MAX_RESULT_CHARACTERS = 100_000 _CANCEL_POLL_SECONDS = 0.1 _CANCEL_GRACE_SECONDS = 2.0 +# A local server that accepts the connection and then never answers leaves the +# child, and the parent waiting on it, blocked forever. Generous enough not to cut +# a long legitimate run short; 0 restores the unbounded wait. +_DEFAULT_TIMEOUT_SECONDS = 1800.0 def _required_env(name: str) -> str: @@ -38,6 +42,18 @@ def _required_env(name: str) -> str: return value +def _timeout_seconds() -> float: + """Wall-clock cap on one child run; 0 or unparsable means wait forever.""" + raw = os.environ.get("UNSLOTH_CLAUDE_SUBAGENT_TIMEOUT") + if raw is None or not raw.strip(): + return _DEFAULT_TIMEOUT_SECONDS + try: + parsed = float(raw.strip()) + except ValueError: + return _DEFAULT_TIMEOUT_SECONDS + return parsed if parsed > 0 else 0.0 + + def _bounded(text: str) -> str: if len(text) <= _MAX_RESULT_CHARACTERS: return text @@ -153,6 +169,17 @@ def run_local_agent( "--output-format", "json", "--no-session-persistence", + # Strip human-blocking tools so the child runs unattended. Only the read-only + # child's writers bite today, since a --print child is never offered the plan + # or prompt tools; those are listed anyway so a version that starts offering + # them cannot stall the subagent. Bash is denied read-only side because plan + # mode gates it through the same local model, which is not a write barrier. + "--disallowedTools", + ( + "AskUserQuestion,EnterPlanMode,Edit,Write,NotebookEdit,Bash" + if read_only + else "AskUserQuestion,EnterPlanMode,ExitPlanMode" + ), "--append-system-prompt", _SUBAGENT_PLAN_INSTRUCTIONS if read_only else _SUBAGENT_INSTRUCTIONS, f"Task: {task}", @@ -185,6 +212,8 @@ def run_local_agent( [executable, *command[1:]], **popen_kwargs, ) + deadline = _timeout_seconds() + started_at = time.monotonic() try: while True: try: @@ -194,6 +223,13 @@ def run_local_agent( if cancel_event.is_set(): _stop_child(process) raise RuntimeError("The local Claude agent was cancelled.") + waited = time.monotonic() - started_at + if deadline and waited > deadline: + _stop_child(process) + raise RuntimeError( + f"The local Claude agent produced nothing after {waited:.0f}s. " + "The local server is likely wedged; check that a model is loaded." + ) except BaseException: if process.poll() is None: _stop_child(process) diff --git a/unsloth_cli/commands/start.py b/unsloth_cli/commands/start.py index 079c7850e5..85e303215b 100644 --- a/unsloth_cli/commands/start.py +++ b/unsloth_cli/commands/start.py @@ -4,6 +4,7 @@ """`unsloth start` — launch a coding agent against a running Unsloth server.""" import atexit +import base64 import contextlib import json import os @@ -2052,6 +2053,32 @@ def _opencode_subagent_inline_config(path: Path, permission: dict) -> dict: return inline +def _b64_path(path: Path) -> str: + """Path as base64, so it can cross a shell without being expanded.""" + return base64.b64encode(str(path).encode("utf-8")).decode("ascii") + + +_CLAUDE_PLAN_GATE_SCRIPT = '''\ +"""Deny the editing agent while the parent session is in plan mode.""" +import json, sys + +try: + mode = (json.load(sys.stdin) or {}).get("permission_mode") +except Exception: + sys.exit(0) # fail open: a hook error must never block the parent session +if mode == "plan": + print(json.dumps({"hookSpecificOutput": { + "hookEventName": "PreToolUse", + "permissionDecision": "deny", + "permissionDecisionReason": ( + "Plan mode is active. Call the read-only Unsloth plan agent " + "(unsloth_plan_agent) instead of unsloth_agent." + ), + }})) +sys.exit(0) +''' + + def write_claude_subagent_plugin(path: Path, server_env: dict) -> Path: """Write a session plugin that exposes the local Claude child through MCP.""" plugin = path / "unsloth-local-agent" @@ -2094,6 +2121,52 @@ def write_claude_subagent_plugin(path: Path, server_env: dict) -> Path: } }, ) + # Claude already refuses the editing tool in plan mode, since it advertises + # readOnlyHint false. This PreToolUse hook replaces that dead end with a reason + # naming the read-only tool to call instead. Skipped under the WSL bridge, where + # the gate is a Linux path but the hook would run beside the Windows claude. + gate = plugin / "hooks" / "plan_gate.py" + if command == "wsl.exe": + # A persisted plugin dir may still hold a gate from an earlier non-WSL run. + for stale in (gate, plugin / "hooks" / "hooks.json"): + stale.unlink(missing_ok = True) + else: + _write_private_text(gate, _CLAUDE_PLAN_GATE_SCRIPT) + _write_private_json( + plugin / "hooks" / "hooks.json", + { + "hooks": { + "PreToolUse": [ + { + "matcher": _CLAUDE_SUBAGENT_TOOL, + "hooks": [ + { + "type": "command", + # Run through runpy rather than handing the path to + # the interpreter: a missing gate is then an + # ordinary traceback (exit 1, fails open) instead + # of exit 2, which Claude treats as a blocking + # error and would deny the tool in every mode. + # The path is base64'd because this string goes + # through a shell: a temp root holding $(..) or a + # backtick expands under sh, %VAR% under cmd, and + # the gate then silently fails open. base64's + # alphabet has no metacharacter in either. + "command": ( + f'"{sys.executable}" -c ' + f'"import base64,runpy; runpy.run_path(' + f"base64.b64decode('{_b64_path(gate)}').decode())\"" + ), + # A hook with no timeout stalls the parent for as + # long as it hangs; measured unbounded past 400s. + "timeout": 10, + } + ], + } + ] + } + }, + ) skill = plugin / "skills" / "local-agent" / "SKILL.md" skill.parent.mkdir(parents = True, exist_ok = True, mode = 0o700) skill.write_text( diff --git a/unsloth_cli/tests/test_claude_plan_gate.py b/unsloth_cli/tests/test_claude_plan_gate.py new file mode 100644 index 0000000000..85336c8dfe --- /dev/null +++ b/unsloth_cli/tests/test_claude_plan_gate.py @@ -0,0 +1,179 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Deterministic plan-mode routing for the local Claude subagent. + +SKILL.md asks the parent model to pick the read-only tool in plan mode, which a +small local model can forget. The generated plugin also ships a PreToolUse hook +that reads permission_mode directly, so the editing agent is denied by rule. +""" + +from __future__ import annotations + +import json +import subprocess +import sys + +import pytest + +from unsloth_cli.commands import start + + +def _plugin(tmp_path): + return start.write_claude_subagent_plugin(tmp_path, {"UNSLOTH_CLAUDE_SUBAGENT_MODEL": "m"}) + + +def _run_gate(script, payload): + return subprocess.run( + [sys.executable, str(script)], + input = payload, + capture_output = True, + text = True, + timeout = 30, + ) + + +def test_plugin_registers_a_pretooluse_hook_on_the_editing_tool(tmp_path): + plugin = _plugin(tmp_path) + + hooks = json.loads((plugin / "hooks" / "hooks.json").read_text())["hooks"]["PreToolUse"] + + [entry] = hooks + # Only the destructive tool is gated; the read-only agent stays reachable. + assert entry["matcher"] == start._CLAUDE_SUBAGENT_TOOL + assert start._CLAUDE_SUBAGENT_PLAN_TOOL not in json.dumps(hooks) + [hook] = entry["hooks"] + assert hook["type"] == "command" + assert sys.executable in hook["command"] + # The interpreter is quoted: unquoted, any space in the path splits the command. + assert f'"{sys.executable}"' in hook["command"] + # The gate path rides as base64, never as a literal the shell can expand. + encoded = start._b64_path(plugin / "hooks" / "plan_gate.py") + assert encoded in hook["command"] + assert str(plugin / "hooks" / "plan_gate.py") not in hook["command"] + # A hook with no timeout stalls the parent for as long as it hangs. + assert 0 < hook["timeout"] <= 30 + + +def test_gate_script_is_written_and_compiles(tmp_path): + plugin = _plugin(tmp_path) + gate = plugin / "hooks" / "plan_gate.py" + + compile(gate.read_text(), str(gate), "exec") # syntax-valid as shipped + + +def test_gate_denies_the_editing_tool_in_plan_mode(tmp_path): + gate = _plugin(tmp_path) / "hooks" / "plan_gate.py" + + result = _run_gate(gate, json.dumps({"permission_mode": "plan"})) + + assert result.returncode == 0 + output = json.loads(result.stdout)["hookSpecificOutput"] + assert output["hookEventName"] == "PreToolUse" + assert output["permissionDecision"] == "deny" + # The reason is shown to the model, so it must name the tool to call instead. + assert "unsloth_plan_agent" in output["permissionDecisionReason"] + + +@pytest.mark.parametrize("mode", ["default", "acceptEdits", "bypassPermissions", "dontAsk", "auto"]) +def test_gate_allows_every_non_plan_mode(tmp_path, mode): + gate = _plugin(tmp_path) / "hooks" / "plan_gate.py" + + result = _run_gate(gate, json.dumps({"permission_mode": mode})) + + assert result.returncode == 0 + assert result.stdout.strip() == "" # no decision -> normal permission flow + + +@pytest.mark.parametrize("payload", ["", "not json", "[]", "null", "{}"]) +def test_gate_fails_open_on_unusable_input(tmp_path, payload): + # A hook crash would block the parent session, so anything unparsable allows. + gate = _plugin(tmp_path) / "hooks" / "plan_gate.py" + + result = _run_gate(gate, payload) + + assert result.returncode == 0 + assert result.stdout.strip() == "" + + +def test_plugin_still_writes_the_mcp_server_and_skill(tmp_path): + # The hook is additive; the existing wiring must be untouched. + plugin = _plugin(tmp_path) + + assert (plugin / ".mcp.json").exists() + assert (plugin / "skills" / "local-agent" / "SKILL.md").exists() + assert (plugin / ".claude-plugin" / "plugin.json").exists() + + +def test_wsl_run_clears_a_gate_left_by_an_earlier_windows_run(tmp_path, monkeypatch): + # The plugin dir survives across runs when persisted, so a gate written by a + # Windows run would otherwise be shipped into the distro with an interpreter + # path it cannot execute. + plugin = _plugin(tmp_path) + gate = plugin / "hooks" / "plan_gate.py" + hooks = plugin / "hooks" / "hooks.json" + assert gate.exists() and hooks.exists() + + monkeypatch.setattr(start, "_wsl_windows_executable", lambda _argv: True) + monkeypatch.setenv("WSL_DISTRO_NAME", "Ubuntu") + _plugin(tmp_path) + + assert not gate.exists() + assert not hooks.exists() + + +def test_hook_command_survives_a_missing_gate_and_a_path_with_spaces(tmp_path): + # Handing the path straight to the interpreter makes a missing gate exit 2, + # which Claude treats as a blocking error: the editing tool would then be + # denied in every mode, not just plan. Going through runpy makes it exit 1. + plugin = _plugin(tmp_path / "dir with space") + hook = json.loads((plugin / "hooks" / "hooks.json").read_text()) + command = hook["hooks"]["PreToolUse"][0]["hooks"][0]["command"] + + # Works normally through the real shell path Claude uses. + denied = subprocess.run( + command, + input = json.dumps({"permission_mode": "plan"}), + shell = True, + capture_output = True, + text = True, + timeout = 30, + ) + assert denied.returncode == 0 + assert json.loads(denied.stdout)["hookSpecificOutput"]["permissionDecision"] == "deny" + + (plugin / "hooks" / "plan_gate.py").unlink() + gone = subprocess.run( + command, + input = json.dumps({"permission_mode": "default"}), + shell = True, + capture_output = True, + text = True, + timeout = 30, + ) + assert gone.returncode != 2, "exit 2 blocks the tool in every mode" + assert gone.stdout.strip() == "" + + +@pytest.mark.parametrize("hostile", ["sub$(echo X)", "tick`echo X`", "var$HOME", "pct%TEMP%pct"]) +def test_gate_survives_shell_metacharacters_in_its_path(tmp_path, hostile): + # The hook command is run by a shell. A temp root holding these expands under + # sh (or cmd, for %VAR%) before Python sees the path, so the gate is not found + # and exits 1, which fails open and silently drops the routing message. + plugin = _plugin(tmp_path / hostile) + command = json.loads((plugin / "hooks" / "hooks.json").read_text())["hooks"]["PreToolUse"][0][ + "hooks" + ][0]["command"] + + denied = subprocess.run( + command, + input = json.dumps({"permission_mode": "plan"}), + shell = True, + capture_output = True, + text = True, + timeout = 60, + ) + + assert denied.returncode == 0, denied.stderr + decision = json.loads(denied.stdout)["hookSpecificOutput"]["permissionDecision"] + assert decision == "deny" diff --git a/unsloth_cli/tests/test_claude_subagent_mcp.py b/unsloth_cli/tests/test_claude_subagent_mcp.py index 568dc76ff5..2155ca050d 100644 --- a/unsloth_cli/tests/test_claude_subagent_mcp.py +++ b/unsloth_cli/tests/test_claude_subagent_mcp.py @@ -15,6 +15,16 @@ import pytest import unsloth_cli.claude_subagent_mcp as bridge +def _stub_env(monkeypatch, tmp_path): + """Minimum env + claude lookup for driving run_local_agent under a fake Popen.""" + monkeypatch.setenv("UNSLOTH_CLAUDE_SUBAGENT_BASE_URL", "http://127.0.0.1:8888") + monkeypatch.setenv("UNSLOTH_CLAUDE_SUBAGENT_API_KEY", "sk-unsloth-test") + monkeypatch.setenv("UNSLOTH_CLAUDE_SUBAGENT_MODEL", "unsloth/model-GGUF:Q4_K_M") + monkeypatch.setenv("CLAUDE_PROJECT_DIR", str(tmp_path)) + monkeypatch.setattr(bridge.shutil, "which", lambda _: "/usr/local/bin/claude") + monkeypatch.setattr(bridge, "_claude_flags", lambda model: ["--settings", "{}"]) + + def test_protocol_lists_and_calls_local_agent(): initialized = bridge._response( {"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {}}, @@ -229,6 +239,8 @@ def test_local_child_uses_unsloth_without_overwriting_parent_auth( assert command[:3] == ["/usr/local/bin/claude", "--model", "unsloth/model-GGUF:Q4_K_M"] assert command[command.index("--permission-mode") + 1] == permission assert "--no-session-persistence" in command + disallowed = command[command.index("--disallowedTools") + 1] + assert disallowed == "AskUserQuestion,EnterPlanMode,ExitPlanMode" assert captured["cwd"] == str(tmp_path) assert captured["stdin"] is bridge.subprocess.DEVNULL assert captured["stdout"] is bridge.subprocess.PIPE @@ -275,6 +287,10 @@ def test_read_only_local_child_uses_plan_mode(monkeypatch, tmp_path): assert bridge.run_local_agent("plan this", read_only = True) == "PLAN_OK" command = captured["command"] assert command[command.index("--permission-mode") + 1] == "plan" + disallowed = command[command.index("--disallowedTools") + 1] + assert disallowed == "AskUserQuestion,EnterPlanMode,Edit,Write,NotebookEdit,Bash" + # Bash matters: plan mode routes it through a classifier served by this same + # local model, so without the deny a "read-only" child can still write files. prompt = command[command.index("--append-system-prompt") + 1] assert "read-only local coding subagent" in prompt @@ -403,3 +419,37 @@ def test_stop_child_kills_survivors_after_leader_exit(monkeypatch, tmp_path): def test_result_parser_accepts_diagnostics_before_json(): output = "connector warning\n" + json.dumps({"is_error": False, "result": "OK"}) assert bridge._result_text(output) == "OK" + + +def test_child_is_stopped_when_it_produces_nothing_before_the_deadline(monkeypatch, tmp_path): + # A local server that accepts and never answers used to block the child, and + # the parent waiting on it, indefinitely. Measured past 400s before this. + monkeypatch.setenv("UNSLOTH_CLAUDE_SUBAGENT_TIMEOUT", "0.3") + _stub_env(monkeypatch, tmp_path) + stopped = [] + + class _Hanging: + returncode = None + + def communicate(self, timeout = None): + raise subprocess.TimeoutExpired("claude", timeout) + + def poll(self): + return None + + monkeypatch.setattr(bridge, "_stop_child", lambda proc: stopped.append(proc)) + monkeypatch.setattr(bridge.subprocess, "Popen", lambda *a, **k: _Hanging()) + + with pytest.raises(RuntimeError, match = "produced nothing"): + bridge.run_local_agent("hello") + assert stopped, "a timed-out child must be killed, not left running" + + +def test_timeout_can_be_disabled(monkeypatch): + monkeypatch.setenv("UNSLOTH_CLAUDE_SUBAGENT_TIMEOUT", "0") + assert bridge._timeout_seconds() == 0.0 + for bad in ("", " ", "abc"): + monkeypatch.setenv("UNSLOTH_CLAUDE_SUBAGENT_TIMEOUT", bad) + assert bridge._timeout_seconds() == bridge._DEFAULT_TIMEOUT_SECONDS + monkeypatch.delenv("UNSLOTH_CLAUDE_SUBAGENT_TIMEOUT") + assert bridge._timeout_seconds() == bridge._DEFAULT_TIMEOUT_SECONDS From b9585d0f627cbb727eaa9b457ba4e336ad743ab0 Mon Sep 17 00:00:00 2001 From: Vineeth Sai Varikuntla Date: Mon, 27 Jul 2026 04:21:27 -0700 Subject: [PATCH 038/132] Keep the newer-mapper probe from replacing the installed FP8 mappers (#7478) * Keep the newer-mapper probe from replacing the installed FP8 mappers get_model_name calls _get_new_mapper() whenever a name misses the local tables, only to answer whether a newer Unsloth would support it. That helper fetches mapper.py from main, prefixes INT_TO_FLOAT_MAPPER, FLOAT_TO_INT_MAPPER and MAP_TO_UNSLOTH_16bit with NEW_, and execs the result into globals(). The slice starts at __INT_TO_FLOAT_MAPPER, so it also carries FLOAT_TO_FP8_BLOCK_MAPPER, FLOAT_TO_FP8_ROW_MAPPER, the _add_* helpers and the builder's loop variables, and none of those are renamed. Exec'ing into globals() therefore rebinds the two FP8 tables that loader_utils imported from the installed mapper, so every later get_model_name(..., load_in_fp8 = ...) in the process resolves through main's table instead of the installed one. The probe deliberately does not adopt the new 4bit mappers (it raises NotImplementedError asking the user to upgrade), so silently adopting the new FP8 ones is inconsistent, and it also leaves loader_utils and mapper disagreeing about the same tables. Reaching it needs nothing unusual: any org/model name absent from the tables triggers the fetch. Exec into a throwaway namespace and read the three mappers out of it, so the probe stays a read and the installed mappings are left alone. Signed-off-by: Vineeth Sai * Hand the fetched FP8 tables back from the probe instead of dropping them Isolating the exec stopped the probe corrupting the installed FP8 tables, but it also removed the only reason the probe ever saw the fetched ones: the _resolve_with_mappers call still read FLOAT_TO_FP8_BLOCK_MAPPER and FLOAT_TO_FP8_ROW_MAPPER off the module globals. A newly added FP8 repo would then miss both the installed tables and the probe, so an older install would stop raising the upgrade NotImplementedError for it. Return the two fetched tables and let _resolve_with_mappers take them as optional arguments, defaulting to the installed ones. The probe now answers for new FP8 repos without writing over what the installed version resolves. _get_new_mapper returns five tables now, so the two existing stubs in test_get_model_name.py and test_bad_mappings_redirect.py are updated to match. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Signed-off-by: Vineeth Sai Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- tests/test_bad_mappings_redirect.py | 2 +- tests/test_get_model_name.py | 3 +- tests/test_new_mapper_no_global_leak.py | 99 +++++++++++++++++++++++++ unsloth/models/loader_utils.py | 50 ++++++++++--- 4 files changed, 141 insertions(+), 13 deletions(-) create mode 100644 tests/test_new_mapper_no_global_leak.py diff --git a/tests/test_bad_mappings_redirect.py b/tests/test_bad_mappings_redirect.py index 49dab2d98b..7ae9ddbb4f 100644 --- a/tests/test_bad_mappings_redirect.py +++ b/tests/test_bad_mappings_redirect.py @@ -26,7 +26,7 @@ def _load_get_model_name(): namespace = dict(mapper_ns) namespace["SUPPORTS_FOURBIT"] = True namespace["_env_says_offline"] = lambda: True - namespace["_get_new_mapper"] = lambda: ({}, {}, {}) + namespace["_get_new_mapper"] = lambda: ({}, {}, {}, {}, {}) wanted = {"__get_model_name", "_resolve_with_mappers", "get_model_name"} for node in tree.body: diff --git a/tests/test_get_model_name.py b/tests/test_get_model_name.py index 33ad316d88..bbd5bfacb1 100644 --- a/tests/test_get_model_name.py +++ b/tests/test_get_model_name.py @@ -6,7 +6,8 @@ from unsloth.models.mapper import FLOAT_TO_INT_MAPPER, MAP_TO_UNSLOTH_16bit def _no_remote_mapper(): - return {}, {}, {} + # int_to_float, float_to_int, map_to_16bit, fp8_block, fp8_row + return {}, {}, {}, {}, {} class TestGetModelName(unittest.TestCase): diff --git a/tests/test_new_mapper_no_global_leak.py b/tests/test_new_mapper_no_global_leak.py new file mode 100644 index 0000000000..5ec00fba5a --- /dev/null +++ b/tests/test_new_mapper_no_global_leak.py @@ -0,0 +1,99 @@ +"""Regression test for ``_get_new_mapper`` leaking into ``loader_utils`` globals. + +``get_model_name`` calls ``_get_new_mapper()`` whenever a name misses the local +tables, purely to answer "would a newer Unsloth support this?". It fetches +``mapper.py`` from GitHub main, prefixes the three mappers it wants with +``NEW_``, and ``exec``s the result into ``globals()``. + +The slice starts at ``__INT_TO_FLOAT_MAPPER``, so it also carries +``FLOAT_TO_FP8_BLOCK_MAPPER``/``FLOAT_TO_FP8_ROW_MAPPER`` and the two +``_add_*`` helpers, and those names are NOT renamed. Exec'ing into +``globals()`` therefore rebinds the FP8 tables that ``loader_utils`` imported +from the installed ``mapper``, so every later ``get_model_name(..., +load_in_fp8 = ...)`` in the process resolves through GitHub main's table +instead of the installed one. The probe is supposed to read, not to swap the +installed mappings out from under the caller. + +``loader_utils`` imports torch, so ast-extract ``_get_new_mapper`` and run it +against a stubbed ``requests`` rather than importing unsloth (which needs a GPU). +""" + +import ast +import os +import sys +import types + +_MODELS = os.path.join(os.path.dirname(__file__), os.pardir, "unsloth", "models") + + +def _mapper_source(): + with open(os.path.join(_MODELS, "mapper.py"), encoding = "utf-8") as f: + return f.read() + + +def _extract_get_new_mapper(namespace): + with open(os.path.join(_MODELS, "loader_utils.py"), encoding = "utf-8") as f: + tree = ast.parse(f.read()) + for node in tree.body: + if isinstance(node, ast.FunctionDef) and node.name == "_get_new_mapper": + exec(compile(ast.Module([node], []), node.name, "exec"), namespace) + return namespace["_get_new_mapper"] + raise AssertionError("_get_new_mapper not found in loader_utils.py") + + +class _FakeResponse: + def __init__(self, text): + self.text = text + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + +def _install_fake_requests(monkeypatch, text): + module = types.ModuleType("requests") + module.get = lambda url, timeout = None: _FakeResponse(text) + monkeypatch.setitem(sys.modules, "requests", module) + + +def test_get_new_mapper_does_not_rebind_the_installed_fp8_tables(monkeypatch): + _install_fake_requests(monkeypatch, _mapper_source()) + + installed = {} + exec(compile(_mapper_source(), "mapper.py", "exec"), installed) + block = installed["FLOAT_TO_FP8_BLOCK_MAPPER"] + row = installed["FLOAT_TO_FP8_ROW_MAPPER"] + assert block and row, "the installed FP8 tables should not be empty" + + # Stand in for loader_utils' module globals, which import the FP8 tables. + namespace = {"FLOAT_TO_FP8_BLOCK_MAPPER": block, "FLOAT_TO_FP8_ROW_MAPPER": row} + get_new_mapper = _extract_get_new_mapper(namespace) + + int_to_float, float_to_int, map_to_16bit, fp8_block, fp8_row = get_new_mapper() + + # _get_new_mapper swallows every exception and returns empty dicts, so assert + # it actually ran before trusting anything below. + assert int_to_float and float_to_int and map_to_16bit, "the fetch/exec path did not run" + + # the probe has to hand the FETCHED fp8 tables back, or a newly added fp8 repo would + # miss both the installed tables and the probe and skip the upgrade message + assert fp8_block and fp8_row + assert fp8_block is not block and fp8_row is not row + + assert namespace["FLOAT_TO_FP8_BLOCK_MAPPER"] is block + assert namespace["FLOAT_TO_FP8_ROW_MAPPER"] is row + + +def test_get_new_mapper_leaves_no_helpers_behind(monkeypatch): + _install_fake_requests(monkeypatch, _mapper_source()) + + namespace = {} + get_new_mapper = _extract_get_new_mapper(namespace) + before = set(namespace) + + assert all(get_new_mapper()), "the fetch/exec path did not run" + + leaked = set(namespace) - before + assert not leaked, f"_get_new_mapper leaked {sorted(leaked)} into its module globals" diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py index 7fd8cd66b4..8214adc0bf 100644 --- a/unsloth/models/loader_utils.py +++ b/unsloth/models/loader_utils.py @@ -191,19 +191,39 @@ def _get_new_mapper(): .replace("MAP_TO_UNSLOTH_16bit", "NEW_MAP_TO_UNSLOTH_16bit") ) - exec(new_mapper, globals()) + # Exec into a throwaway namespace, never globals(). The slice also carries + # FLOAT_TO_FP8_BLOCK_MAPPER / FLOAT_TO_FP8_ROW_MAPPER, the _add_* helpers + # and the builder's loop variables, so exec'ing into globals() would swap + # the FP8 tables this module imported from the installed mapper for the + # ones on GitHub main. This is only a probe for "would a newer Unsloth + # support this name?", so it must not change what the installed version + # resolves; the fetched FP8 tables are returned for the probe to use + # instead of being written over the installed ones. + namespace = {} + exec(new_mapper, namespace) return ( - NEW_INT_TO_FLOAT_MAPPER, - NEW_FLOAT_TO_INT_MAPPER, - NEW_MAP_TO_UNSLOTH_16bit, + namespace["NEW_INT_TO_FLOAT_MAPPER"], + namespace["NEW_FLOAT_TO_INT_MAPPER"], + namespace["NEW_MAP_TO_UNSLOTH_16bit"], + namespace["FLOAT_TO_FP8_BLOCK_MAPPER"], + namespace["FLOAT_TO_FP8_ROW_MAPPER"], ) except: - return {}, {}, {} + return {}, {}, {}, {}, {} def _resolve_with_mappers( - model_name, load_in_4bit, load_in_fp8, int_to_float, float_to_int, map_to_unsloth_16bit + model_name, + load_in_4bit, + load_in_fp8, + int_to_float, + float_to_int, + map_to_unsloth_16bit, + fp8_block = None, + fp8_row = None, ): + # fp8_block/fp8_row default to the installed tables; the newer-mapper probe passes the + # fetched ones so it can answer for new FP8 repos without rebinding the installed ones. return __get_model_name( model_name = model_name, load_in_4bit = load_in_4bit, @@ -211,8 +231,8 @@ def _resolve_with_mappers( FLOAT_TO_INT_MAPPER = float_to_int, MAP_TO_UNSLOTH_16bit = map_to_unsloth_16bit, load_in_fp8 = load_in_fp8, - FLOAT_TO_FP8_BLOCK_MAPPER = FLOAT_TO_FP8_BLOCK_MAPPER, - FLOAT_TO_FP8_ROW_MAPPER = FLOAT_TO_FP8_ROW_MAPPER, + FLOAT_TO_FP8_BLOCK_MAPPER = FLOAT_TO_FP8_BLOCK_MAPPER if fp8_block is None else fp8_block, + FLOAT_TO_FP8_ROW_MAPPER = FLOAT_TO_FP8_ROW_MAPPER if fp8_row is None else fp8_row, ) @@ -252,9 +272,13 @@ def get_model_name( and not _env_says_offline() # offline: skip the remote (raw GitHub) mapper refresh ): # Try checking if a new Unsloth version allows it! - NEW_INT_TO_FLOAT_MAPPER, NEW_FLOAT_TO_INT_MAPPER, NEW_MAP_TO_UNSLOTH_16bit = ( - _get_new_mapper() - ) + ( + NEW_INT_TO_FLOAT_MAPPER, + NEW_FLOAT_TO_INT_MAPPER, + NEW_MAP_TO_UNSLOTH_16bit, + NEW_FP8_BLOCK_MAPPER, + NEW_FP8_ROW_MAPPER, + ) = _get_new_mapper() upgraded_model_name = _resolve_with_mappers( model_name = model_name, load_in_4bit = load_in_4bit, @@ -262,6 +286,10 @@ def get_model_name( int_to_float = NEW_INT_TO_FLOAT_MAPPER, float_to_int = NEW_FLOAT_TO_INT_MAPPER, map_to_unsloth_16bit = NEW_MAP_TO_UNSLOTH_16bit, + # the fp8 probe has to look at the FETCHED tables too, or a new fp8 repo would + # miss both here and in the installed tables and skip the upgrade message + fp8_block = NEW_FP8_BLOCK_MAPPER, + fp8_row = NEW_FP8_ROW_MAPPER, ) if upgraded_model_name is not None: raise NotImplementedError( From 2ab4b744ac68d9e99bcc4fc444e5413dd3ffbce0 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 27 Jul 2026 04:35:17 -0700 Subject: [PATCH 039/132] Studio: admission control on /v1/messages, slot pool that tracks --parallel (#7436) * Studio: admission control on /v1/messages, slot pool that tracks --parallel /v1/chat/completions was gated by the llama admission queue but /v1/messages was not, so an Anthropic client could oversubscribe llama-server's slots and stall the backend. Wire the same queue into all six /v1/messages dispatch sites, and rework the queue itself into an explicit slot pool. - Queue keyed by base_url, so both API surfaces share one pool of slots. - Waiting is unbounded by default instead of timing out; the wait line is sized at 16 x the serving slots so it follows --parallel. - Neutral UNSLOTH_LLAMA_ADMISSION_* env names, legacy UNSLOTH_OPENAI_COMPAT_* spellings still honored. - Passthrough retries once against a respawned llama-server, which comes back on a new ephemeral port. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix over-admission on capacity shrink and restore the stream cancel contract Review of the previous commit turned up two real regressions plus smaller gaps. - Pool sizing looked only at free slot ids, so when capacity shrank while slots were held (an unload resets effective_parallel_slots to 1) a freed low id was handed out even though the holdovers already met the new ceiling. Count every held slot against capacity instead. A 1-slot backend could run 4 generations. - The streaming wrapper closed the monitored body with aclose(), delivering GeneratorExit where _SameTaskStreamingResponse deliberately throws CancelledError. The monitor entry was never finalized, so it leaked as "running" for the process lifetime and cancel_event was never set. Close through the shared helper so cancellation reaches the handler. - Finalize the monitor when a stream is abandoned before its body starts, and when a queued non-streaming request is cancelled (which also leaked the un-awaited generation coroutine). - Floor the scaled wait line at 64, so a 1-slot backend keeps the depth it had before scaling existed instead of dropping from 64 to 16. - Use the canonical Anthropic type map: a full queue is 429 rate_limit_error, which SDKs back off on; overloaded_error is 529. - Treat non-positive max_queue/queue_per_slot as unbounded rather than "reject everything", and reclaim the slot if a waiter's event loop is gone. Tests: regression tests for both defects, verified to fail without the fix. Adds env coverage for QUEUE_PER_SLOT and the legacy fallbacks, a structural check that all six dispatch sites stay admission-wrapped, and clears the new env var in the isolation fixtures. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Run the response pre-start cleanup when a queued stream is abandoned From automated review of the earlier commits. _anthropic_passthrough_stream enters its _TrackedCancel eagerly and relies on the stream's finally to exit it, but aclose() on an async generator that never started is a no-op, so that finally never runs. Admission made this reachable: a client that disconnects while queued leaves the cancel id registered in _CANCEL_REGISTRY forever. - Give the passthrough response an unstarted_cleanup hook that exits the tracker, via a new optional arg on _sse_streaming_response. - Chain to that hook from the admission wrapper rather than replacing it, and run it when the wrapper gives up before the body started. - Defer to an in-progress MTP fallback instead of respawning underneath it; only the first caller gets True from _maybe_recover_from_mtp_crash. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Restore Python 3.9 support, and stop the floor overriding an explicit setting Second review round. The first item is a real break shipped by the earlier commits, the rest are correctness and contract fixes. - dataclass(slots = True) and int.bit_count() are both 3.10+, but the package declares requires-python >=3.9 and CI only runs 3.12, so nothing caught it. Importing the module raised TypeError on 3.9, taking down the whole backend, not just admission. Drop the dataclass slots and track the popcount in a counter. A test now asserts neither API comes back. - The queue-depth floor applied even when an operator set QUEUE_PER_SLOT explicitly, so asking for a shallow line silently got 64 and, with no queue timeout, callers blocked instead of failing fast. The floor now only backs the default multiplier. - Never let a failing close strand a slot: closing runs in its own try so the release always happens. A lost slot shrinks the pool permanently. - Close the generation coroutine when reserving fails for any reason, not only on a full queue. - Exit the passthrough cancel tracker if the client drops while the opening SSE lines are still being sent; those yields sit outside the teardown try. - snapshot.free now reports what a caller could actually take, so the admission log cannot show free slots next to queued requests after a shrink. - Correct the class docstring: the wait line is bounded by default, not unlimited. Document that abandoning wait() requires cancel(), and pin the thread assumption in _deliver_lease. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Make the leak guards real, and cover the untested admission branches Third review round, which attacked the previous round's tests by reverting each fix. Two guards turned out to be hollow. - The pre-start cleanup chain could be severed with the suite still green: the existing test drove the generator finally, never the response hook. A real pre-start disconnect leaked the passthrough cancel tracker permanently. Replaced with a test that runs the response hook and asserts _CANCEL_REGISTRY is empty; verified against both ways of reintroducing the leak. - The structural check only asserted the unstarted_cleanup keyword was present, so passing a literal None passed it while leaking. It now asserts the hook is actually built. - test_shares_queue_with_openai_by_base_url never touched the OpenAI helper; it was a duplicate under a misleading name. It now reserves through the same helper /v1/chat/completions uses, so it fails if either surface ever derives a different key. That is the PR's central shared-queue claim. - Cover the passthrough dispatch site, 499 on disconnect-while-queued, and the streaming admission timeout. Four of six sites previously had only an AST node count behind them. - Clear admission env in the autouse fixture rather than per test: an ambient canonical name silently beat the legacy name a test was exercising. - Loosen the wall-clock assertion, which guarded against serialising on the uncontended path, not against a slow runner. - The class docstring claimed a global concurrency cap; Studio's own chat endpoint does not reserve, so it is not one. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Keep dataclass slots on 3.10+ via a version gate Dropping slots = True for 3.9 gave it up everywhere, including the 3.12 CI runs and every supported interpreter but one. Gate it instead: _SLOTS is {"slots": True} on 3.10+ and empty below, unpacked into each dataclass. The AST scan now requires the unpack rather than merely forbidding a literal slots keyword, so a dataclass added later cannot quietly lose slots. Added a test that the gate matches the running interpreter, since a gate that never applies is worse than no gate. Verified the 3.9 branch by forcing _SLOTS empty and reloading: the full admission suite passes either way. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix two slot leaks, and cover the guards that had no test Third review round, three reviewers working independently on the admission core, the route wiring, and whether the PR regresses anything it is not about. Leaks: - cancel() made the same call_soon_threadsafe as _grant_waiters_locked but without its RuntimeError guard. Routes cancel from finally blocks, so a closed loop masked their exception and skipped the release, stranding the slot and pinning is_idle() false so the queue was never evicted either. - The pre-start cleanup released the slot after an await that can raise BaseException, which is swallowed upstream. Nested it in a finally, as the streaming and OpenAI paths already do. An unparseable QUEUE_PER_SLOT dropped the 64 floor while falling back to the default multiplier, quietly giving a 1-slot backend a 16-deep line. Explicit now means it parsed. Guards that were correct but had no test. Each was reverted, confirmed the suite stayed green, then covered and confirmed red: - the slot released when stream setup raises, which is the reachable one: count_chat_tokens is a blocking call to llama-server, so a dead backend raises after the slot is taken and before a body exists to release it - coro.close() on a cancelled queued request, the api_monitor.fail that distinguishes an admission timeout from a client hang-up, the MTP fallback short-circuit, and the BaseException guard around the opening stream lines - the queue-full test asserted a type string OpenAI's 429 also uses, so it passed against an OpenAI envelope. It now pins the Anthropic shape. Anthropic requests were invisible in the admission log while sharing the pool with chat completions, so the same events are logged there with a mode. Renamed the helper to match, since it is no longer OpenAI-only. Corrected two comments that described behaviour the code does not have: the slot is taken when the streaming response is built, not when the body starts iterating, and the pool is not a cap on every generation, since /v1/completions, Studio's chat endpoint and RAG captioning all reach llama-server directly. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Cover the admission telemetry, and drop a dead helper Fourth review round. No bugs found in the code this time; the finding was that most of the previous commit's telemetry had no test. Only queue-full was asserted, so removing any of the other four log calls left the suite green. All five are covered now, each verified by removing its call and confirming only its own test reds. Fixing the first attempt turned up a test bug of my own: the log line carries a queued=N field, so asserting "queued" in the message matched every admission log ever emitted. It asserts the event name now. Also covered two guards that were correct but unguarded: waiters whose futures die out of band stop counting against the queue depth, and a newcomer cannot barge past a parked waiter. The second is pinned as behaviour rather than as the `if not self._waiters` check, because that check cannot actually change the outcome: _take_slot_locked consults _can_admit_locked anyway, so either alone refuses the newcomer. The test fails only if both go. _optional_positive_int_env lost its last caller when the env parsing was rewritten last round. Removed. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .../backend/core/inference/llama_admission.py | 274 ++++- studio/backend/routes/inference.py | 409 ++++++-- .../backend/tests/test_anthropic_admission.py | 973 ++++++++++++++++++ .../backend/tests/test_anthropic_messages.py | 11 + .../test_anthropic_passthrough_respawn.py | 262 +++++ studio/backend/tests/test_llama_admission.py | 537 +++++++++- 6 files changed, 2355 insertions(+), 111 deletions(-) create mode 100644 studio/backend/tests/test_anthropic_admission.py create mode 100644 studio/backend/tests/test_anthropic_passthrough_respawn.py diff --git a/studio/backend/core/inference/llama_admission.py b/studio/backend/core/inference/llama_admission.py index b6a939c87b..1a9ae04b0e 100644 --- a/studio/backend/core/inference/llama_admission.py +++ b/studio/backend/core/inference/llama_admission.py @@ -13,37 +13,85 @@ from __future__ import annotations import asyncio import os +import sys import threading from collections import deque from dataclasses import dataclass from typing import Deque, Optional -ADMISSION_CONTROL_ENV = "UNSLOTH_OPENAI_COMPAT_ADMISSION_CONTROL" -ADMISSION_QUEUE_TIMEOUT_ENV = "UNSLOTH_OPENAI_COMPAT_ADMISSION_QUEUE_TIMEOUT" -ADMISSION_KEEPALIVE_INTERVAL_ENV = "UNSLOTH_OPENAI_COMPAT_ADMISSION_KEEPALIVE_INTERVAL" -ADMISSION_MAX_QUEUE_ENV = "UNSLOTH_OPENAI_COMPAT_ADMISSION_MAX_QUEUE" +# dataclass(slots = True) halves per-instance overhead. Measured as perf-neutral +# here, not a speed win: it costs a little on construction and gains it back on +# access. It is 3.10+ and this package declares >=3.9, so gate it rather than +# dropping it outright. Empty on 3.9 means a plain dataclass. +_SLOTS = {"slots": True} if sys.version_info >= (3, 10) else {} + + +ADMISSION_CONTROL_ENV = "UNSLOTH_LLAMA_ADMISSION_CONTROL" +ADMISSION_QUEUE_TIMEOUT_ENV = "UNSLOTH_LLAMA_ADMISSION_QUEUE_TIMEOUT" +ADMISSION_KEEPALIVE_INTERVAL_ENV = "UNSLOTH_LLAMA_ADMISSION_KEEPALIVE_INTERVAL" +ADMISSION_MAX_QUEUE_ENV = "UNSLOTH_LLAMA_ADMISSION_MAX_QUEUE" +ADMISSION_QUEUE_PER_SLOT_ENV = "UNSLOTH_LLAMA_ADMISSION_QUEUE_PER_SLOT" + +# The UNSLOTH_OPENAI_COMPAT_* spellings predate this queue being shared with the +# Anthropic /v1/messages route (same llama-server slots). Still honored; the +# neutral name above wins when both are set. +_LEGACY_ENV = { + ADMISSION_CONTROL_ENV: "UNSLOTH_OPENAI_COMPAT_ADMISSION_CONTROL", + ADMISSION_QUEUE_TIMEOUT_ENV: "UNSLOTH_OPENAI_COMPAT_ADMISSION_QUEUE_TIMEOUT", + ADMISSION_KEEPALIVE_INTERVAL_ENV: "UNSLOTH_OPENAI_COMPAT_ADMISSION_KEEPALIVE_INTERVAL", + ADMISSION_MAX_QUEUE_ENV: "UNSLOTH_OPENAI_COMPAT_ADMISSION_MAX_QUEUE", +} DEFAULT_ADMISSION_ENABLED = True +# None: a queued request waits for its slot indefinitely rather than timing out. DEFAULT_ADMISSION_QUEUE_TIMEOUT_S = None DEFAULT_ADMISSION_KEEPALIVE_INTERVAL_S = 5.0 -DEFAULT_ADMISSION_MAX_QUEUE = 64 +# None: no absolute cap, the wait line is sized from the pool instead. +DEFAULT_ADMISSION_MAX_QUEUE = None +# Wait line = 16 x the serving slots, so it tracks --parallel (4 slots -> 64 +# waiters, 8 -> 128). Purely a memory guard; waiting itself is never timed out. +DEFAULT_ADMISSION_QUEUE_PER_SLOT = 16 +# Floor for the scaled line, so a 1-slot backend (plain `unsloth studio`, or any +# load downshifted to fit VRAM) keeps the depth it had before scaling existed +# rather than dropping to 16 and rejecting callers that used to queue. +DEFAULT_ADMISSION_MIN_QUEUE = 64 -@dataclass(frozen = True) +@dataclass(frozen = True, **_SLOTS) class LlamaAdmissionConfig: enabled: bool = DEFAULT_ADMISSION_ENABLED queue_timeout_s: Optional[float] = DEFAULT_ADMISSION_QUEUE_TIMEOUT_S keepalive_interval_s: float = DEFAULT_ADMISSION_KEEPALIVE_INTERVAL_S max_queue: Optional[int] = DEFAULT_ADMISSION_MAX_QUEUE + queue_per_slot: Optional[int] = DEFAULT_ADMISSION_QUEUE_PER_SLOT + # Unconditional floor on the scaled line. The env path clears it when the + # operator sets QUEUE_PER_SLOT, so only the default multiplier is floored. + min_queue: Optional[int] = DEFAULT_ADMISSION_MIN_QUEUE + + def queue_limit(self, capacity: int) -> Optional[int]: + """How many callers may line up for a pool of ``capacity`` slots. + + An explicit ``max_queue`` wins; otherwise the line scales with the slots + so it follows ``--parallel``. The default multiplier is floored, so a + 1-slot backend does not end up shallower than it was before scaling. None + (or any non-positive setting) means an unbounded line. + """ + if self.max_queue is not None: + return self.max_queue if self.max_queue > 0 else None + if not self.queue_per_slot or self.queue_per_slot <= 0: + return None + scaled = self.queue_per_slot * max(1, capacity) + return max(self.min_queue, scaled) if self.min_queue else scaled -@dataclass(frozen = True) +@dataclass(frozen = True, **_SLOTS) class LlamaAdmissionSnapshot: key: str capacity: int active: int queued: int + free: int = 0 class LlamaAdmissionError(Exception): @@ -69,8 +117,17 @@ class LlamaAdmissionCancelled(LlamaAdmissionError): pass -def _bool_env(name: str, default: bool) -> bool: +def _raw_env(name: str) -> Optional[str]: + """Value for a canonical name, falling back to its legacy spelling.""" value = os.environ.get(name) + if value is None or not value.strip(): + legacy = _LEGACY_ENV.get(name) + value = os.environ.get(legacy) if legacy else None + return value + + +def _bool_env(name: str, default: bool) -> bool: + value = _raw_env(name) if value is None or not value.strip(): return default value = value.strip().lower() @@ -82,7 +139,7 @@ def _bool_env(name: str, default: bool) -> bool: def _optional_positive_float_env(name: str, default: Optional[float]) -> Optional[float]: - value = os.environ.get(name) + value = _raw_env(name) if value is None or not value.strip(): return default try: @@ -93,7 +150,7 @@ def _optional_positive_float_env(name: str, default: Optional[float]) -> Optiona def _positive_float_env(name: str, default: float) -> float: - value = os.environ.get(name) + value = _raw_env(name) if value is None or not value.strip(): return default try: @@ -103,19 +160,38 @@ def _positive_float_env(name: str, default: float) -> float: return parsed if parsed > 0 else default -def _optional_positive_int_env(name: str, default: Optional[int]) -> Optional[int]: - value = os.environ.get(name) - if value is None or not value.strip(): - return default +def _queue_limits_from_env() -> tuple[Optional[int], Optional[int], Optional[int]]: + """(max_queue, queue_per_slot, min_queue) from the environment. + + An absolute MAX_QUEUE wins outright; MAX_QUEUE=0 asks for an unbounded line. + Unset leaves the per-slot multiplier in charge (itself 0 for unbounded). The + floor applies only to the default multiplier: setting QUEUE_PER_SLOT means + the operator wants that exact depth, however shallow. + """ + # Explicit means it parsed, not just that something was set: a typo falls back + # to the default multiplier, so it has to keep the default's floor too. + raw_per_slot = _raw_env(ADMISSION_QUEUE_PER_SLOT_ENV) try: - parsed = int(value.strip()) + per_slot = int((raw_per_slot or "").strip()) except ValueError: - return default - return parsed if parsed > 0 else None + per_slot, min_queue = DEFAULT_ADMISSION_QUEUE_PER_SLOT, DEFAULT_ADMISSION_MIN_QUEUE + else: + per_slot, min_queue = (per_slot if per_slot > 0 else None), None + raw = _raw_env(ADMISSION_MAX_QUEUE_ENV) + if raw is None or not raw.strip(): + return None, per_slot, min_queue + try: + parsed = int(raw.strip()) + except ValueError: + return None, per_slot, min_queue + return (parsed, None, None) if parsed > 0 else (None, None, None) def llama_admission_config_from_env() -> LlamaAdmissionConfig: + max_queue, queue_per_slot, min_queue = _queue_limits_from_env() return LlamaAdmissionConfig( + queue_per_slot = queue_per_slot, + min_queue = min_queue, enabled = _bool_env(ADMISSION_CONTROL_ENV, DEFAULT_ADMISSION_ENABLED), queue_timeout_s = _optional_positive_float_env( ADMISSION_QUEUE_TIMEOUT_ENV, @@ -125,14 +201,11 @@ def llama_admission_config_from_env() -> LlamaAdmissionConfig: ADMISSION_KEEPALIVE_INTERVAL_ENV, DEFAULT_ADMISSION_KEEPALIVE_INTERVAL_S, ), - max_queue = _optional_positive_int_env( - ADMISSION_MAX_QUEUE_ENV, - DEFAULT_ADMISSION_MAX_QUEUE, - ), + max_queue = max_queue, ) -@dataclass +@dataclass(**_SLOTS) class _Waiter: loop: asyncio.AbstractEventLoop future: asyncio.Future @@ -141,11 +214,23 @@ class _Waiter: class LlamaAdmissionLease: - def __init__(self, queue: Optional["LlamaAdmissionQueue"]): + __slots__ = ("_queue", "_slot", "_released", "_release_lock") + + def __init__( + self, + queue: Optional["LlamaAdmissionQueue"], + slot: Optional[int] = None, + ): self._queue = queue + self._slot = slot self._released = False self._release_lock = threading.Lock() + @property + def slot(self) -> Optional[int]: + """Pool slot this lease holds, or None when admission is disabled.""" + return self._slot + def release(self) -> None: queue = None with self._release_lock: @@ -154,7 +239,7 @@ class LlamaAdmissionLease: self._released = True queue = self._queue if queue is not None: - queue.release() + queue.release(self._slot) async def __aenter__(self) -> "LlamaAdmissionLease": return self @@ -164,6 +249,8 @@ class LlamaAdmissionLease: class LlamaAdmissionReservation: + __slots__ = ("_queue", "_lease", "_waiter", "snapshot") + def __init__( self, *, @@ -195,6 +282,13 @@ class LlamaAdmissionReservation: return self._lease async def wait(self, timeout_s: float) -> Optional[LlamaAdmissionLease]: + """Wait up to ``timeout_s`` for a slot. + + A timeout leaves this reservation queued so the caller can poll again. + Any exit that abandons the wait for good must call ``cancel()``, or the + slot granted later is delivered to a future nobody reads and is never + released. + """ lease = self.lease_nowait() if lease is not None: return lease @@ -229,35 +323,80 @@ class LlamaAdmissionReservation: class LlamaAdmissionQueue: + """A fixed pool of generation slots for one llama-server, plus a FIFO wait line. + + The pool mirrors llama-server's own ``--parallel`` slots: ``capacity`` slot ids + are each either free or held by exactly one caller. A caller that finds every + slot busy waits in arrival order and is handed the next slot to free, so no + caller is starved. This bounds only the callers that reserve: chat completions + and messages do, while /v1/completions, Studio's own chat endpoint and RAG + captioning all reach llama-server directly, so it is not a global cap. + Waiting is unbounded in time by default (``queue_timeout_s`` + None); the wait line itself is bounded, and only how many may line up before + new arrivals are rejected. By default that is ``16 x slots`` floored at 64, + not unlimited: an unbounded line takes ``max_queue`` or ``queue_per_slot`` + set to 0. See ``LlamaAdmissionConfig.queue_limit``. + """ + + __slots__ = ("key", "_lock", "_capacity", "_free", "_in_use", "_held", "_waiters") + def __init__(self, key: str): self.key = key self._lock = threading.Lock() - self._active = 0 self._capacity = 1 + self._free: list[int] = [0] + # Held slots as a bitmask: one int instead of a set, so the pool costs the + # same whether it is idle or saturated. _held is its popcount, kept as a + # counter because int.bit_count() is 3.10+ and this package targets 3.9. + self._in_use = 0 + self._held = 0 self._waiters: Deque[_Waiter] = deque() + def _resize_pool_locked(self, capacity: int) -> None: + # Slots past a shrunk capacity retire when their holder releases them. + if capacity == self._capacity: + return + self._capacity = capacity + self._free = [slot for slot in range(capacity) if not self._in_use >> slot & 1] + + def _can_admit_locked(self) -> bool: + # Slots still held above a shrunk capacity keep occupying the backend, so + # count every held slot against the ceiling, not just the ids below it. + return bool(self._free) and self._held < self._capacity + + def _take_slot_locked(self) -> Optional[int]: + if not self._can_admit_locked(): + return None + slot = self._free.pop() + self._in_use |= 1 << slot + self._held += 1 + return slot + def reserve(self, *, capacity: int, config: LlamaAdmissionConfig) -> LlamaAdmissionReservation: capacity = max(1, int(capacity or 1)) if not config.enabled: return LlamaAdmissionReservation( queue = None, lease = LlamaAdmissionLease(None), - snapshot = LlamaAdmissionSnapshot(self.key, capacity, 0, 0), + snapshot = LlamaAdmissionSnapshot(self.key, capacity, 0, 0, capacity), ) loop = asyncio.get_running_loop() with self._lock: - self._capacity = capacity - self._prune_waiters_locked() + self._resize_pool_locked(capacity) self._grant_waiters_locked() - if self._active < self._capacity and not self._waiters: - self._active += 1 - return LlamaAdmissionReservation( - queue = self, - lease = LlamaAdmissionLease(self), - snapshot = self._snapshot_locked(), - ) - if config.max_queue is not None and len(self._waiters) >= config.max_queue: + if not self._waiters: + slot = self._take_slot_locked() + if slot is not None: + # No snapshot here: callers read it through snapshot_now(), + # which re-reads the queue, so building one per admitted + # request would be pure allocation on the hot path. + return LlamaAdmissionReservation( + queue = self, + lease = LlamaAdmissionLease(self, slot), + ) + limit = config.queue_limit(self._capacity) + if limit is not None and self._live_waiters_locked() >= limit: raise LlamaAdmissionQueueFull( "llama-server generation queue is full", snapshot = self._snapshot_locked(), @@ -270,13 +409,20 @@ class LlamaAdmissionQueue: return LlamaAdmissionReservation( queue = self, waiter = waiter, - snapshot = self._snapshot_locked(), ) - def release(self) -> None: + def _release_slot_locked(self, slot: Optional[int]) -> None: + # A slot id at or past a shrunk capacity retires instead of returning. + if slot is None or not self._in_use >> slot & 1: + return + self._in_use &= ~(1 << slot) + self._held -= 1 + if slot < self._capacity: + self._free.append(slot) + + def release(self, slot: Optional[int]) -> None: with self._lock: - if self._active > 0: - self._active -= 1 + self._release_slot_locked(slot) self._grant_waiters_locked() def cancel(self, waiter: _Waiter) -> None: @@ -291,7 +437,13 @@ class LlamaAdmissionQueue: lease_to_release = waiter.granted_lease waiter.granted_lease = None if not waiter.future.done(): - waiter.loop.call_soon_threadsafe(waiter.future.cancel) + try: + waiter.loop.call_soon_threadsafe(waiter.future.cancel) + except RuntimeError: + # Loop gone. Routes call cancel() from finally blocks, so + # raising here would both mask their exception and skip the + # release below, stranding the slot for the process lifetime. + pass if lease_to_release is not None: lease_to_release.release() @@ -303,20 +455,30 @@ class LlamaAdmissionQueue: def is_idle(self) -> bool: with self._lock: self._prune_waiters_locked() - return self._active == 0 and not self._waiters + return self._in_use == 0 and not self._waiters def _grant_waiters_locked(self) -> None: - self._prune_waiters_locked() - while self._waiters and self._active < self._capacity: + # Dead waiters are skipped as they are popped, so no prune is needed here. + while self._waiters and self._can_admit_locked(): waiter = self._waiters.popleft() if waiter.cancelled or waiter.future.done(): continue - self._active += 1 - lease = LlamaAdmissionLease(self) + slot = self._take_slot_locked() + lease = LlamaAdmissionLease(self, slot) waiter.granted_lease = lease - waiter.loop.call_soon_threadsafe(self._deliver_lease, waiter, lease) + try: + waiter.loop.call_soon_threadsafe(self._deliver_lease, waiter, lease) + except RuntimeError: + # Waiter's loop is gone. Reclaim the slot; leaving the bit set + # would strand it, since _free is rebuilt from the bitmask. + waiter.granted_lease = None + self._release_slot_locked(slot) def _deliver_lease(self, waiter: _Waiter, lease: LlamaAdmissionLease) -> None: + # Runs on the waiter's own loop thread, which is also the only thread that + # cancels that reservation, so waiter state is safe to touch unlocked here. + # release() may be called from any thread, but only reaches this via + # call_soon_threadsafe. Cancelling off-loop would need this under _lock. if waiter.cancelled or waiter.future.done(): waiter.granted_lease = None if not waiter.future.done(): @@ -331,16 +493,32 @@ class LlamaAdmissionQueue: lease.release() def _prune_waiters_locked(self) -> None: + # Rebuilding the deque on every reserve/release dominated the hot path, so + # only pay it when a waiter actually died out of band (an externally + # cancelled future); cancel() already drops its own waiter eagerly. + for waiter in self._waiters: + if waiter.cancelled or waiter.future.done(): + break + else: + return self._waiters = deque( waiter for waiter in self._waiters if not waiter.cancelled and not waiter.future.done() ) + def _live_waiters_locked(self) -> int: + self._prune_waiters_locked() + return len(self._waiters) + def _snapshot_locked(self) -> LlamaAdmissionSnapshot: return LlamaAdmissionSnapshot( key = self.key, capacity = self._capacity, - active = self._active, + active = self._held, queued = len(self._waiters), + # What another caller could actually take, so the admission log never + # shows free slots next to queued requests: after a shrink, ids below + # the new capacity can be free while holdovers still fill the ceiling. + free = min(len(self._free), max(0, self._capacity - self._held)), ) diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 06911fd866..cf95e743bf 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -387,7 +387,7 @@ def _raise_unsupported_n(path_label: str) -> None: _raise_unsupported_openai_parameter("n", f"n > 1 is not supported for {path_label}.") -def _sse_streaming_response(content) -> StreamingResponse: +def _sse_streaming_response(content, *, unstarted_cleanup = None) -> StreamingResponse: """A ``text/event-stream`` response with the standard SSE headers used by every streaming path here: no client/proxy caching, no proxy buffering, and a one-shot connection. Two callers build their response inline instead: the @@ -409,6 +409,7 @@ def _sse_streaming_response(content) -> StreamingResponse: "Connection": "close", "X-Accel-Buffering": "no", }, + unstarted_cleanup = unstarted_cleanup, ) @@ -1141,7 +1142,7 @@ def _openai_admission_request_path(request: Optional[Request]) -> Optional[str]: return None -def _openai_admission_log( +def _llama_admission_log( event: str, reservation: Optional[LlamaAdmissionReservation] = None, *, @@ -1159,13 +1160,15 @@ def _openai_admission_log( wait_ms = int(max(0.0, time.monotonic() - wait_started_at) * 1000) log = getattr(logger, level, logger.debug) log( - "openai admission %s: mode=%s path=%s completion_id=%s capacity=%s active=%s queued=%s wait_ms=%s", + "llama admission %s: mode=%s path=%s completion_id=%s " + "pool=%s/%s free=%s queued=%s wait_ms=%s", event, mode, _openai_admission_request_path(request), completion_id, - getattr(snapshot, "capacity", None), getattr(snapshot, "active", None), + getattr(snapshot, "capacity", None), + getattr(snapshot, "free", None), getattr(snapshot, "queued", None), wait_ms, ) @@ -1189,6 +1192,23 @@ def _openai_admission_http_exception(exc: Exception, *, status_code: int) -> HTT ) +def _anthropic_admission_http_exception(exc: Exception, *, status_code: int) -> HTTPException: + """Anthropic-shaped error for an admission reject/timeout/cancel (429/503/499).""" + snapshot = getattr(exc, "snapshot", None) + message = str(exc) + if snapshot is not None: + message = ( + f"{message} " + f"(active={snapshot.active}, queued={snapshot.queued}, capacity={snapshot.capacity})" + ) + # Types come from ANTHROPIC_TYPE_BY_STATUS (429 -> rate_limit_error, which is + # what Anthropic SDKs back off on); overloaded_error is reserved for 529. + return HTTPException( + status_code = status_code, + detail = anthropic_error_body(message, status = status_code), + ) + + def _openai_admission_timeout_error( reservation: LlamaAdmissionReservation, ) -> LlamaAdmissionTimeout: @@ -1494,6 +1514,24 @@ class _SameTaskStreamingResponse(StreamingResponse): await self.background() +async def _release_unstarted_anthropic_stream(iterator, prior_cleanup) -> None: + """Close a stream whose body never started, running the response's own + pre-start hook. aclose() on an unstarted async generator is a no-op, so its + finally never runs and anything the builder acquired eagerly (the passthrough + cancel tracker) would leak without the hook.""" + aclose = getattr(iterator, "aclose", None) + if aclose is not None: + try: + await aclose() + except Exception: + pass + if prior_cleanup is not None: + try: + await prior_cleanup() + except Exception: + pass + + def _tracked_cancel_unstarted_cleanup(tracker): """unstarted_cleanup that exits ``tracker`` on a pre-start disconnect, when the generator's finally (which normally exits it) never runs.""" @@ -8286,7 +8324,7 @@ async def openai_chat_completions( llama_backend = llama_backend, ) except LlamaAdmissionQueueFull as exc: - _openai_admission_log( + _llama_admission_log( "queue-full", snapshot = exc.snapshot, request = request, @@ -8524,7 +8562,7 @@ async def openai_chat_completions( admission_wait_started_at = None if stream_lease is None: admission_wait_started_at = time.monotonic() - _openai_admission_log( + _llama_admission_log( "queued", reservation, request = request, @@ -8549,7 +8587,7 @@ async def openai_chat_completions( yield wait_item continue lease = wait_item - _openai_admission_log( + _llama_admission_log( "granted-after-wait", reservation, request = request, @@ -8580,7 +8618,7 @@ async def openai_chat_completions( cancelled = stream_cancelled, ) except LlamaAdmissionTimeout as exc: - _openai_admission_log( + _llama_admission_log( "timeout", reservation, request = request, @@ -8594,7 +8632,7 @@ async def openai_chat_completions( _openai_admission_error_body(exc, status_code = 503) ) except LlamaAdmissionCancelled: - _openai_admission_log( + _llama_admission_log( "cancelled-before-upstream", reservation, request = request, @@ -8705,7 +8743,7 @@ async def openai_chat_completions( try: if reservation.lease_nowait() is None: admission_wait_started_at = time.monotonic() - _openai_admission_log( + _llama_admission_log( "queued", reservation, request = request, @@ -8720,7 +8758,7 @@ async def openai_chat_completions( cancel_event = cancel_event, ) if admission_wait_started_at is not None: - _openai_admission_log( + _llama_admission_log( "granted-after-wait", reservation, request = request, @@ -8791,7 +8829,7 @@ async def openai_chat_completions( _tracker.__exit__(None, None, None) raise except LlamaAdmissionTimeout as exc: - _openai_admission_log( + _llama_admission_log( "timeout", reservation, request = request, @@ -8806,7 +8844,7 @@ async def openai_chat_completions( _tracker.__exit__(None, None, None) raise _openai_admission_http_exception(exc, status_code = 503) except LlamaAdmissionCancelled as exc: - _openai_admission_log( + _llama_admission_log( "cancelled-before-upstream", reservation, request = request, @@ -8886,7 +8924,7 @@ async def openai_chat_completions( ) except LlamaAdmissionQueueFull as exc: _tracker.__exit__(None, None, None) - _openai_admission_log( + _llama_admission_log( "queue-full", snapshot = exc.snapshot, request = request, @@ -9058,7 +9096,7 @@ async def openai_chat_completions( admission_wait_started_at = None if stream_lease is None: admission_wait_started_at = time.monotonic() - _openai_admission_log( + _llama_admission_log( "queued", reservation, request = request, @@ -9083,7 +9121,7 @@ async def openai_chat_completions( yield wait_item continue lease = wait_item - _openai_admission_log( + _llama_admission_log( "granted-after-wait", reservation, request = request, @@ -9114,7 +9152,7 @@ async def openai_chat_completions( cancelled = stream_cancelled, ) except LlamaAdmissionTimeout as exc: - _openai_admission_log( + _llama_admission_log( "timeout", reservation, request = request, @@ -9128,7 +9166,7 @@ async def openai_chat_completions( _openai_admission_error_body(exc, status_code = 503) ) except LlamaAdmissionCancelled: - _openai_admission_log( + _llama_admission_log( "cancelled-before-upstream", reservation, request = request, @@ -9184,7 +9222,7 @@ async def openai_chat_completions( llama_backend = llama_backend, ) except LlamaAdmissionQueueFull as exc: - _openai_admission_log( + _llama_admission_log( "queue-full", snapshot = exc.snapshot, request = request, @@ -9203,7 +9241,7 @@ async def openai_chat_completions( try: if reservation.lease_nowait() is None: admission_wait_started_at = time.monotonic() - _openai_admission_log( + _llama_admission_log( "queued", reservation, request = request, @@ -9218,7 +9256,7 @@ async def openai_chat_completions( cancel_event = cancel_event, ) if admission_wait_started_at is not None: - _openai_admission_log( + _llama_admission_log( "granted-after-wait", reservation, request = request, @@ -9240,7 +9278,7 @@ async def openai_chat_completions( _tracker.__exit__(None, None, None) raise except LlamaAdmissionTimeout as exc: - _openai_admission_log( + _llama_admission_log( "timeout", reservation, request = request, @@ -9255,7 +9293,7 @@ async def openai_chat_completions( _tracker.__exit__(None, None, None) raise _openai_admission_http_exception(exc, status_code = 503) except LlamaAdmissionCancelled as exc: - _openai_admission_log( + _llama_admission_log( "cancelled-before-upstream", reservation, request = request, @@ -11713,7 +11751,7 @@ async def _responses_stream( llama_backend = llama_backend, ) except LlamaAdmissionQueueFull as exc: - _openai_admission_log( + _llama_admission_log( "queue-full", snapshot = exc.snapshot, request = request, @@ -12571,7 +12609,7 @@ async def _responses_stream( try: if lease is None: admission_wait_started_at = time.monotonic() - _openai_admission_log( + _llama_admission_log( "queued", reservation, request = request, @@ -12589,7 +12627,7 @@ async def _responses_stream( yield wait_item continue lease = wait_item - _openai_admission_log( + _llama_admission_log( "granted-after-wait", reservation, request = request, @@ -12621,7 +12659,7 @@ async def _responses_stream( cancelled = stream_cancelled, ) except LlamaAdmissionTimeout as exc: - _openai_admission_log( + _llama_admission_log( "timeout", reservation, request = request, @@ -12633,7 +12671,7 @@ async def _responses_stream( api_monitor.fail(monitor_id, str(exc)) yield _responses_admission_failed_sse(exc, status_code = 503) except LlamaAdmissionCancelled: - _openai_admission_log( + _llama_admission_log( "cancelled-before-upstream", reservation, request = request, @@ -13233,12 +13271,206 @@ async def anthropic_messages( cancel_event, ) + # ── Admission control ───────────────────────────────────── + # Bound concurrent llama-server generations to the backend's serving slots via a + # FIFO queue keyed by base_url (shared with /v1/chat/completions, same slots). + # Excess requests queue; a streaming waiter gets SSE keep-alives, the queue 429s + # once full. Mirrors the OpenAI passthrough admission wiring. Streaming takes the + # slot when the response is built and drops it when the body finishes or is + # abandoned; the non-stream path holds it across the single awaited generation. + _anthropic_admission_mode = "anthropic_stream" if payload.stream else "anthropic_nonstream" + + async def _admitted_anthropic_stream( + orig_body, + reservation, + admission_config, + stream_lease, + prior_cleanup = None, + ): + lease = stream_lease + stream_cancelled = False + body_started = False + wait_started_at = None + try: + if lease is None: + wait_started_at = time.monotonic() + _llama_admission_log( + "queued", + reservation, + request = request, + mode = _anthropic_admission_mode, + ) + async for wait_item in _openai_admission_wait_stream_chunks( + reservation, + admission_config, + request = request, + cancel_event = cancel_event, + ): + if isinstance(wait_item, str): + yield wait_item + continue + lease = wait_item + break + _llama_admission_log( + "granted-after-wait", + reservation, + request = request, + mode = _anthropic_admission_mode, + wait_started_at = wait_started_at, + ) + if lease is None: + return + body_started = True + async for chunk in orig_body: + yield chunk + except asyncio.CancelledError: + # Must reach the monitored generator as CancelledError, not aclose's + # GeneratorExit, or its handler never finalizes the monitor entry. + stream_cancelled = True + raise + except LlamaAdmissionTimeout as exc: + api_monitor.fail(monitor_id, str(exc)) + _llama_admission_log( + "timeout", + reservation, + request = request, + mode = _anthropic_admission_mode, + wait_started_at = wait_started_at, + level = "warning", + ) + yield build_anthropic_sse_event( + "error", + anthropic_error_body(str(exc), status = 503), + ) + except LlamaAdmissionCancelled: + _llama_admission_log( + "cancelled-before-upstream", + reservation, + request = request, + mode = _anthropic_admission_mode, + wait_started_at = wait_started_at, + ) + return + finally: + # Closing can raise (a raw body re-raises CancelledError after + # teardown), and a slot lost that way never comes back: with no queue + # timeout the pool just shrinks and later callers wait forever. Keep + # the release in its own finally, as the /responses wiring does. + try: + if body_started: + await _close_openai_admitted_stream_iterator( + orig_body, + cancelled = stream_cancelled, + ) + else: + # Gave up while queued: the monitored body never ran, so nothing + # downstream finalizes the entry or exits the response's tracker. + api_monitor.finish(monitor_id, "cancelled") + await _release_unstarted_anthropic_stream(orig_body, prior_cleanup) + finally: + if lease is not None: + lease.release() + else: + reservation.cancel() + + async def _admitted_anthropic(coro): + try: + reservation, admission_config = _openai_llama_admission_reserve( + request = request, llama_backend = llama_backend + ) + except LlamaAdmissionQueueFull as exc: + coro.close() + api_monitor.fail(monitor_id, str(exc)) + _llama_admission_log( + "queue-full", + snapshot = getattr(exc, "snapshot", None), + request = request, + mode = _anthropic_admission_mode, + level = "warning", + ) + raise _anthropic_admission_http_exception(exc, status_code = 429) + except BaseException: + # Reserving never awaited the generation, so close it rather than + # leave an un-awaited coroutine behind. + coro.close() + raise + + if payload.stream: + stream_lease = reservation.lease_nowait() + # Set up the stream (token count + tracker enter) and surface a pre-response + # cancel now, exactly as the un-admitted path did; the upstream generation is + # deferred to body iteration, so the slot is only held while tokens flow. + try: + # Token counting calls llama-server, so a dead backend raises here + # with the slot already taken. cancel() covers both cases: it + # releases the lease if one was granted, else drops the waiter. + monitored = await _monitored_anthropic(coro) + except BaseException: + reservation.cancel() + raise + orig_body = getattr(monitored, "body_iterator", None) + if orig_body is None: + reservation.cancel() + return monitored + + # Replacing body_iterator would strand the response's own pre-start + # hook (the passthrough uses one to exit its cancel tracker), so chain + # to it instead of clobbering it. + prior_cleanup = getattr(monitored, "_unstarted_cleanup", None) + + async def _unstarted_cleanup() -> None: + # The body never ran, so nothing else closes out the monitor entry. + api_monitor.finish(monitor_id, "cancelled") + try: + await _release_unstarted_anthropic_stream(orig_body, prior_cleanup) + finally: + # A BaseException here is swallowed upstream, so releasing + # outside the finally would shrink the pool silently. + reservation.cancel() + + monitored.body_iterator = _admitted_anthropic_stream( + orig_body, reservation, admission_config, stream_lease, prior_cleanup + ) + monitored._unstarted_cleanup = _unstarted_cleanup + return monitored + + lease = None + try: + lease = await _wait_for_openai_admission_non_streaming( + reservation, + admission_config, + request = request, + cancel_event = cancel_event, + ) + monitored = await _monitored_anthropic(coro) + return monitored + except LlamaAdmissionTimeout as exc: + coro.close() + api_monitor.fail(monitor_id, str(exc)) + raise _anthropic_admission_http_exception(exc, status_code = 503) + except LlamaAdmissionCancelled as exc: + coro.close() + api_monitor.finish(monitor_id, "cancelled") + raise _anthropic_admission_http_exception(exc, status_code = 499) + except BaseException: + # Cancelled while queued (shutdown, outer task cancel): the generation + # coroutine was never awaited, so close it rather than leak it. + if lease is None: + coro.close() + api_monitor.finish(monitor_id, "cancelled") + raise + finally: + if lease is not None: + lease.release() + else: + reservation.cancel() + # ── Client-side pass-through path ───────────────────────── if client_tools: openai_tools = openai_client_tools if payload.stream: - return await _monitored_anthropic( + return await _admitted_anthropic( _anthropic_passthrough_stream( request, cancel_event, @@ -13262,7 +13494,7 @@ async def anthropic_messages( auto_heal_tool_calls = payload.auto_heal_tool_calls, ) ) - return await _monitored_anthropic( + return await _admitted_anthropic( _anthropic_passthrough_non_streaming( llama_backend, openai_messages, @@ -13367,7 +13599,7 @@ async def anthropic_messages( ) if payload.stream: - return await _monitored_anthropic( + return await _admitted_anthropic( _anthropic_tool_stream( request, cancel_event, @@ -13380,7 +13612,7 @@ async def anthropic_messages( disable_parallel_tool_use = _disable_parallel, ) ) - return await _monitored_anthropic( + return await _admitted_anthropic( _anthropic_tool_non_streaming( _run_tool_gen, message_id, @@ -13407,7 +13639,7 @@ async def anthropic_messages( ) if payload.stream: - return await _monitored_anthropic( + return await _admitted_anthropic( _anthropic_plain_stream( request, cancel_event, @@ -13418,7 +13650,7 @@ async def anthropic_messages( openai_messages = openai_messages, ) ) - return await _monitored_anthropic( + return await _admitted_anthropic( _anthropic_plain_non_streaming( _run_plain_gen, message_id, @@ -13930,6 +14162,28 @@ def _build_passthrough_payload( return body +async def _anthropic_passthrough_retry_url(llama_backend, exc): + """Fresh upstream URL after respawning a dead llama-server, else None. + + A crashed server relaunches on a NEW ephemeral port, so a passthrough still + holding the old base_url keeps failing until the next load. Mirrors the + respawn-and-retry in generate_chat_completion. None when an MTP+tensor crash + already scheduled its own recovery, or when nothing needed respawning. + """ + recover = getattr(llama_backend, "_maybe_recover_from_mtp_crash", None) + if recover is not None and recover(exc): + return None + # Only the first caller gets True above; the rest must not respawn the same + # MTP config underneath the fallback that is already reloading without it. + if getattr(llama_backend, "_mtp_runtime_fallback_in_progress", False): + return None + respawn = getattr(llama_backend, "_respawn_if_dead", None) + if respawn is None or not await asyncio.to_thread(respawn): + return None + logger.warning("llama-server was unreachable; respawned it and retrying the passthrough") + return f"{llama_backend.base_url}/v1/chat/completions" + + async def _anthropic_passthrough_stream( request, cancel_event, @@ -13997,8 +14251,15 @@ async def _anthropic_passthrough_stream( openai_tools, disable_parallel_tool_use = disable_parallel_tool_use, ) - for line in emitter.start(message_id, model_name, input_tokens = input_tokens): - yield line + # These yields sit outside the teardown try below, so a disconnect while + # the opening lines are being sent would strand the tracker. __exit__ is + # idempotent, so the normal path still exits once, down there. + try: + for line in emitter.start(message_id, model_name, input_tokens = input_tokens): + yield line + except BaseException: + _tracker.__exit__(None, None, None) + raise # Manage the httpx client, response, AND the aiter_lines() async # generator MANUALLY -- no `async with`, no anonymous iterator. @@ -14033,13 +14294,24 @@ async def _anthropic_passthrough_stream( cancel_watcher = None disconnect_watcher = None try: - req = client.build_request( - "POST", target_url, json = body, headers = {"Connection": "close"} - ) - first_token_deadline = time.monotonic() + _DEFAULT_FIRST_TOKEN_TIMEOUT_S - resp = await _send_stream_with_preheader_cancel( - client, req, cancel_event, request = request - ) + url = target_url + try: + req = client.build_request("POST", url, json = body, headers = {"Connection": "close"}) + first_token_deadline = time.monotonic() + _DEFAULT_FIRST_TOKEN_TIMEOUT_S + resp = await _send_stream_with_preheader_cancel( + client, req, cancel_event, request = request + ) + except httpx.ConnectError as exc: + # Nothing has streamed yet, so a respawned server can be retried once + # on its new port without duplicating output. + url = await _anthropic_passthrough_retry_url(llama_backend, exc) + if url is None: + raise + req = client.build_request("POST", url, json = body, headers = {"Connection": "close"}) + first_token_deadline = time.monotonic() + _DEFAULT_FIRST_TOKEN_TIMEOUT_S + resp = await _send_stream_with_preheader_cancel( + client, req, cancel_event, request = request + ) if resp is None: return @@ -14118,7 +14390,13 @@ async def _anthropic_passthrough_stream( for line in emitter.finish(): yield line - return _sse_streaming_response(_stream()) + # The tracker is entered eagerly above, but _stream()'s finally is what exits + # it. Closing an async generator that never started is a no-op, so hand the + # response a cleanup hook or a pre-start give-up leaks the registry entry. + return _sse_streaming_response( + _stream(), + unstarted_cleanup = _tracked_cancel_unstarted_cleanup(_tracker), + ) async def _anthropic_passthrough_non_streaming( @@ -14158,11 +14436,24 @@ async def _anthropic_passthrough_non_streaming( backend_ctx = llama_backend.context_length, ) - resp = await nonstreaming_client().post( - target_url, - json = body, - timeout = _llama_non_streaming_generation_timeout(), - ) + try: + resp = await nonstreaming_client().post( + target_url, + json = body, + timeout = _llama_non_streaming_generation_timeout(), + ) + except httpx.ConnectError as exc: + # Nothing was returned yet, so retry once against the respawned server's + # new port; the nudge retry below then reuses the same fresh URL. + retry_url = await _anthropic_passthrough_retry_url(llama_backend, exc) + if retry_url is None: + raise + target_url = retry_url + resp = await nonstreaming_client().post( + target_url, + json = body, + timeout = _llama_non_streaming_generation_timeout(), + ) if resp.status_code != 200: raise HTTPException( @@ -14667,7 +14958,7 @@ async def _openai_passthrough_stream( ) except LlamaAdmissionQueueFull as exc: _tracker.__exit__(None, None, None) - _openai_admission_log( + _llama_admission_log( "queue-full", snapshot = exc.snapshot, request = request, @@ -14712,7 +15003,7 @@ async def _openai_passthrough_stream( ) admission_wait_started_at = time.monotonic() - _openai_admission_log( + _llama_admission_log( "queued", reservation, request = request, @@ -14736,7 +15027,7 @@ async def _openai_passthrough_stream( if isinstance(wait_item, str): yield wait_item continue - _openai_admission_log( + _llama_admission_log( "granted-after-wait", reservation, request = request, @@ -14781,7 +15072,7 @@ async def _openai_passthrough_stream( await cleanup() return except LlamaAdmissionTimeout as exc: - _openai_admission_log( + _llama_admission_log( "timeout", reservation, request = request, @@ -14793,7 +15084,7 @@ async def _openai_passthrough_stream( api_monitor.fail(monitor_id, str(exc)) yield _openai_stream_error_sse(_openai_admission_error_body(exc, status_code = 503)) except LlamaAdmissionCancelled: - _openai_admission_log( + _llama_admission_log( "cancelled-before-upstream", reservation, request = request, @@ -15570,7 +15861,7 @@ async def _openai_passthrough_non_streaming( llama_backend = llama_backend, ) except LlamaAdmissionQueueFull as exc: - _openai_admission_log( + _llama_admission_log( "queue-full", snapshot = exc.snapshot, request = request, @@ -15585,7 +15876,7 @@ async def _openai_passthrough_non_streaming( try: if reservation.lease_nowait() is None: admission_wait_started_at = time.monotonic() - _openai_admission_log( + _llama_admission_log( "queued", reservation, request = request, @@ -15599,7 +15890,7 @@ async def _openai_passthrough_non_streaming( cancel_event = cancel_event, ) if admission_wait_started_at is not None: - _openai_admission_log( + _llama_admission_log( "granted-after-wait", reservation, request = request, @@ -15621,7 +15912,7 @@ async def _openai_passthrough_non_streaming( cancel_event = cancel_event, ) except LlamaAdmissionTimeout as exc: - _openai_admission_log( + _llama_admission_log( "timeout", reservation, request = request, @@ -15632,7 +15923,7 @@ async def _openai_passthrough_non_streaming( api_monitor.fail(monitor_id, str(exc)) raise _openai_admission_http_exception(exc, status_code = 503) except LlamaAdmissionCancelled as exc: - _openai_admission_log( + _llama_admission_log( "cancelled-before-upstream", reservation, request = request, diff --git a/studio/backend/tests/test_anthropic_admission.py b/studio/backend/tests/test_anthropic_admission.py new file mode 100644 index 0000000000..de01accd08 --- /dev/null +++ b/studio/backend/tests/test_anthropic_admission.py @@ -0,0 +1,973 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. + +"""Admission-control wiring for the Anthropic /v1/messages endpoint. + +The FIFO queue itself is unit-tested in test_llama_admission.py; here we exercise +how anthropic_messages reserves a slot, queues when the backend is saturated, +streams keep-alives while waiting, releases on completion, and maps rejects to +429/503. Slot occupancy is driven directly through the shared queue (keyed by the +backend base_url) so generation stays fast and no thread has to block. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import gc +import os +import re +import sys +import threading +import time +import warnings +from types import SimpleNamespace + +import httpx +import pytest + +_backend = os.path.join(os.path.dirname(__file__), "..") +sys.path.insert(0, _backend) + +import routes.inference as inf_mod +from routes.inference import ( + _anthropic_passthrough_retry_url, + _anthropic_passthrough_stream, + anthropic_messages, +) +from models.inference import AnthropicMessagesRequest +from core.inference.api_monitor import ApiMonitor +from core.inference.llama_admission import ( + ADMISSION_CONTROL_ENV, + ADMISSION_KEEPALIVE_INTERVAL_ENV, + ADMISSION_MAX_QUEUE_ENV, + ADMISSION_QUEUE_PER_SLOT_ENV, + ADMISSION_QUEUE_TIMEOUT_ENV, + LlamaAdmissionConfig, + get_llama_admission_queue, + reset_llama_admission_queues, +) +from fastapi import HTTPException + +_KEY = "http://llama.admission.test:9999" + + +@pytest.fixture(autouse = True) +def _isolate(monkeypatch): + reset_llama_admission_queues() + monkeypatch.setattr(inf_mod, "api_monitor", ApiMonitor(max_entries = 64)) + monkeypatch.setattr(inf_mod, "_CANCEL_REGISTRY", {}) + for name in ( + ADMISSION_CONTROL_ENV, + ADMISSION_QUEUE_TIMEOUT_ENV, + ADMISSION_KEEPALIVE_INTERVAL_ENV, + ADMISSION_MAX_QUEUE_ENV, + ADMISSION_QUEUE_PER_SLOT_ENV, + # Legacy spellings resolve too, so clear both for isolation. + "UNSLOTH_OPENAI_COMPAT_ADMISSION_CONTROL", + "UNSLOTH_OPENAI_COMPAT_ADMISSION_QUEUE_TIMEOUT", + "UNSLOTH_OPENAI_COMPAT_ADMISSION_KEEPALIVE_INTERVAL", + "UNSLOTH_OPENAI_COMPAT_ADMISSION_MAX_QUEUE", + ): + monkeypatch.delenv(name, raising = False) + yield + reset_llama_admission_queues() + + +class _Request: + def __init__(self, disconnected = False): + self.state = SimpleNamespace() + self.url = SimpleNamespace(path = "/v1/messages") + self.method = "POST" + self._disconnected = disconnected + + async def is_disconnected(self): + return self._disconnected + + +def _install_backend( + monkeypatch, + *, + slots = 1, + base_url = _KEY, + count_tokens = None, +): + def _gen_plain(**_kwargs): + yield "ok" + + def _gen_tools(**_kwargs): + yield {"type": "content", "text": "ok"} + + backend = SimpleNamespace( + is_loaded = True, + is_vision = False, + supports_tools = True, + supports_tool_passthrough = False, + model_identifier = "test-model", + context_length = 2048, + count_chat_tokens = count_tokens or (lambda *a, **k: 2), + generate_chat_completion = _gen_plain, + generate_chat_completion_with_tools = _gen_tools, + effective_parallel_slots = slots, + base_url = base_url, + ) + monkeypatch.setattr(inf_mod, "get_llama_cpp_backend", lambda: backend) + return backend + + +def _payload(**fields) -> AnthropicMessagesRequest: + base = {"max_tokens": 16, "messages": [{"role": "user", "content": "hi"}]} + base.update(fields) + return AnthropicMessagesRequest(**base) + + +def _record_admission_logs(monkeypatch): + """Capture _llama_admission_log output. + + Through the logger rather than caplog: this one is a structlog bound logger, + so it never reaches the stdlib handlers caplog installs. + """ + records = [] + + def _record(level): + return lambda fmt, *args: records.append((level, fmt % args)) + + monkeypatch.setattr( + inf_mod, + "logger", + SimpleNamespace( + debug = _record("debug"), + info = _record("info"), + warning = _record("warning"), + ), + ) + return records + + +def _snapshot(key = _KEY): + return get_llama_admission_queue(key).snapshot() + + +def _occupy(key, capacity, n): + """Hold ``n`` slots on the queue so the next reserve must wait; returns leases.""" + leases = [] + for _ in range(n): + reservation = get_llama_admission_queue(key).reserve( + capacity = capacity, config = LlamaAdmissionConfig() + ) + lease = reservation.lease_nowait() + assert lease is not None + leases.append(lease) + return leases + + +async def _consume(response): + chunks = [] + async for chunk in response.body_iterator: + chunks.append(chunk.decode() if isinstance(chunk, (bytes, bytearray)) else chunk) + return "".join(chunks) + + +# ── Non-streaming ───────────────────────────────────────────── + + +def test_non_streaming_completes_and_releases_slot(monkeypatch): + _install_backend(monkeypatch, slots = 2) + + async def _run(): + response = await anthropic_messages(_payload(), request = _Request(), current_subject = "t") + assert response.status_code == 200 + snap = _snapshot() + assert snap.active == 0 and snap.queued == 0 + + asyncio.run(_run()) + + +def test_non_streaming_queue_full_returns_429(monkeypatch): + monkeypatch.setenv(ADMISSION_MAX_QUEUE_ENV, "1") + _install_backend(monkeypatch, slots = 1) + + async def _run(): + held = _occupy(_KEY, 1, 1) # slot busy + # One waiter fills the max_queue=1; the next reserve rejects. + get_llama_admission_queue(_KEY).reserve( + capacity = 1, config = LlamaAdmissionConfig(max_queue = 1) + ) + with pytest.raises(HTTPException) as exc: + await anthropic_messages(_payload(), request = _Request(), current_subject = "t") + assert exc.value.status_code == 429 + # rate_limit_error is what Anthropic SDKs back off on; overloaded_error is 529. + # The type string alone does not pin the envelope, since OpenAI's 429 uses the + # same word. Assert the shape too, or emitting an OpenAI body still passes. + detail = exc.value.detail + assert detail["type"] == "error" + assert "request_id" in detail + assert set(detail["error"]) == {"type", "message"} + assert detail["error"]["type"] == "rate_limit_error" + for lease in held: + lease.release() + + asyncio.run(_run()) + + +def test_admission_events_are_logged_on_the_anthropic_surface(monkeypatch): + # The OpenAI passthrough logs these with a mode; without the same on /v1/messages + # an operator debugging a slow Anthropic client has nothing to look at, and the + # pool is shared, so it is the same triage. + records = _record_admission_logs(monkeypatch) + monkeypatch.setenv(ADMISSION_MAX_QUEUE_ENV, "1") + _install_backend(monkeypatch, slots = 1) + + async def _run(): + held = _occupy(_KEY, 1, 1) + get_llama_admission_queue(_KEY).reserve( + capacity = 1, config = LlamaAdmissionConfig(max_queue = 1) + ) + with pytest.raises(HTTPException): + await anthropic_messages(_payload(), request = _Request(), current_subject = "t") + for lease in held: + lease.release() + + asyncio.run(_run()) + full = [msg for _level, msg in records if "queue-full" in msg] + assert full, records + assert "llama admission queue-full" in full[0] + assert "mode=anthropic_nonstream" in full[0] + + +def test_streaming_admission_waiting_is_logged(monkeypatch): + # queued and granted-after-wait were both emitted with nothing asserting them. + records = _record_admission_logs(monkeypatch) + monkeypatch.setenv(ADMISSION_KEEPALIVE_INTERVAL_ENV, "0.05") + _install_backend(monkeypatch, slots = 1) + + async def _run(): + held = _occupy(_KEY, 1, 1) + response = await anthropic_messages( + _payload(stream = True), request = _Request(), current_subject = "t" + ) + task = asyncio.create_task(_consume(response)) + await asyncio.sleep(0.15) + for lease in held: + lease.release() + await asyncio.wait_for(task, timeout = 5) + + asyncio.run(_run()) + events = [msg for _level, msg in records if "llama admission" in msg] + # "llama admission queued", not "queued": every line carries a queued=N field, + # so the bare substring matches any admission log at all. + assert any( + "llama admission queued" in m and "mode=anthropic_stream" in m for m in events + ), events + granted = [m for m in events if "granted-after-wait" in m] + assert granted, events + # wait_ms is the point of the event: a grant that reports nothing is useless. + assert re.search(r"wait_ms=\d+", granted[0]), granted + + +def test_streaming_admission_timeout_is_logged(monkeypatch): + records = _record_admission_logs(monkeypatch) + monkeypatch.setenv(ADMISSION_QUEUE_TIMEOUT_ENV, "0.15") + monkeypatch.setenv(ADMISSION_KEEPALIVE_INTERVAL_ENV, "0.05") + _install_backend(monkeypatch, slots = 1) + + async def _run(): + held = _occupy(_KEY, 1, 1) # never released, so the waiter times out + response = await anthropic_messages( + _payload(stream = True), request = _Request(), current_subject = "t" + ) + await _consume(response) + for lease in held: + lease.release() + + asyncio.run(_run()) + timeouts = [msg for level, msg in records if "timeout" in msg and level == "warning"] + assert timeouts, records + assert "mode=anthropic_stream" in timeouts[0] + + +def test_streaming_give_up_while_queued_is_logged(monkeypatch): + # cancelled-before-upstream is the one that tells an operator a client walked + # away rather than the backend being slow. + records = _record_admission_logs(monkeypatch) + monkeypatch.setenv(ADMISSION_KEEPALIVE_INTERVAL_ENV, "0.05") + _install_backend(monkeypatch, slots = 1) + + async def _run(): + held = _occupy(_KEY, 1, 1) + response = await anthropic_messages( + _payload(stream = True), + request = _Request(disconnected = True), + current_subject = "t", + ) + await _consume(response) + for lease in held: + lease.release() + + asyncio.run(_run()) + events = [msg for _level, msg in records if "llama admission" in msg] + assert any("llama admission cancelled-before-upstream" in m for m in events), events + + +def test_non_streaming_times_out_returns_503(monkeypatch): + monkeypatch.setenv(ADMISSION_QUEUE_TIMEOUT_ENV, "0.15") + _install_backend(monkeypatch, slots = 1) + + async def _run(): + held = _occupy(_KEY, 1, 1) # never released -> waiter times out + with pytest.raises(HTTPException) as exc: + await anthropic_messages(_payload(), request = _Request(), current_subject = "t") + assert exc.value.status_code == 503 + for lease in held: + lease.release() + + asyncio.run(_run()) + + +def test_non_streaming_queued_then_admitted(monkeypatch): + _install_backend(monkeypatch, slots = 1) + + async def _run(): + held = _occupy(_KEY, 1, 1) + task = asyncio.create_task( + anthropic_messages(_payload(), request = _Request(), current_subject = "t") + ) + await asyncio.sleep(0.1) + assert _snapshot().queued == 1 # waiting on the busy slot + held[0].release() # free it + response = await asyncio.wait_for(task, timeout = 2) + assert response.status_code == 200 + assert _snapshot().active == 0 and _snapshot().queued == 0 + + asyncio.run(_run()) + + +def test_capacity_enforced_from_effective_parallel_slots(monkeypatch): + _install_backend(monkeypatch, slots = 3) + + async def _run(): + held = _occupy(_KEY, 3, 3) # all 3 slots busy + task = asyncio.create_task( + anthropic_messages(_payload(), request = _Request(), current_subject = "t") + ) + await asyncio.sleep(0.1) + snap = _snapshot() + assert snap.capacity == 3 and snap.active == 3 and snap.queued == 1 + for lease in held: + lease.release() + response = await asyncio.wait_for(task, timeout = 2) + assert response.status_code == 200 + + asyncio.run(_run()) + + +def test_disabled_admission_bypasses_limit(monkeypatch): + monkeypatch.setenv(ADMISSION_CONTROL_ENV, "off") + _install_backend(monkeypatch, slots = 1) + + async def _run(): + held = _occupy(_KEY, 1, 1) # would block if admission were on + response = await asyncio.wait_for( + anthropic_messages(_payload(), request = _Request(), current_subject = "t"), + timeout = 2, + ) + assert response.status_code == 200 + for lease in held: + lease.release() + + asyncio.run(_run()) + + +# ── Streaming ───────────────────────────────────────────────── + + +def test_streaming_completes_and_releases_slot(monkeypatch): + _install_backend(monkeypatch, slots = 1) + + async def _run(): + response = await anthropic_messages( + _payload(stream = True), request = _Request(), current_subject = "t" + ) + blob = await _consume(response) + assert "event: message_start" in blob + assert "event: message_stop" in blob + assert _snapshot().active == 0 and _snapshot().queued == 0 + + asyncio.run(_run()) + + +def test_streaming_emits_keepalives_while_queued_then_streams(monkeypatch): + monkeypatch.setenv(ADMISSION_KEEPALIVE_INTERVAL_ENV, "0.05") + _install_backend(monkeypatch, slots = 1) + + async def _run(): + held = _occupy(_KEY, 1, 1) + response = await anthropic_messages( + _payload(stream = True), request = _Request(), current_subject = "t" + ) + body = response.body_iterator + # First chunk must be a keep-alive comment (slot still busy). + first = await asyncio.wait_for(body.__anext__(), timeout = 2) + first = first.decode() if isinstance(first, (bytes, bytearray)) else first + assert first.startswith(":") # SSE comment keep-alive + held[0].release() # free the slot -> real stream follows + rest = await asyncio.wait_for(_drain(body), timeout = 2) + assert "event: message_start" in rest + assert _snapshot().active == 0 and _snapshot().queued == 0 + + asyncio.run(_run()) + + +def test_streaming_queue_full_returns_429(monkeypatch): + monkeypatch.setenv(ADMISSION_MAX_QUEUE_ENV, "1") + _install_backend(monkeypatch, slots = 1) + + async def _run(): + held = _occupy(_KEY, 1, 1) + get_llama_admission_queue(_KEY).reserve( + capacity = 1, config = LlamaAdmissionConfig(max_queue = 1) + ) + with pytest.raises(HTTPException) as exc: + await anthropic_messages(_payload(stream = True), request = _Request(), current_subject = "t") + assert exc.value.status_code == 429 + for lease in held: + lease.release() + + asyncio.run(_run()) + + +def test_streaming_disconnect_while_queued_frees_slot(monkeypatch): + monkeypatch.setenv(ADMISSION_KEEPALIVE_INTERVAL_ENV, "0.05") + _install_backend(monkeypatch, slots = 1) + + async def _run(): + held = _occupy(_KEY, 1, 1) + response = await anthropic_messages( + _payload(stream = True), request = _Request(), current_subject = "t" + ) + body = response.body_iterator + await asyncio.wait_for(body.__anext__(), timeout = 2) # one keep-alive + assert _snapshot().queued == 1 + await body.aclose() # client goes away mid-wait + held[0].release() + await asyncio.sleep(0.05) + snap = _snapshot() + assert snap.queued == 0 and snap.active == 0 + + asyncio.run(_run()) + + +# ── Shared queue + fairness + speed ─────────────────────────── + + +def test_shares_queue_with_openai_by_base_url(monkeypatch): + """The two API surfaces must land on one pool of the same llama-server slots. + + Reserves through the OpenAI helper the /v1/chat/completions path uses, rather + than poking the queue directly, so this fails if either side ever derives a + different key. + """ + _install_backend(monkeypatch, slots = 1) + + async def _run(): + openai_reservation, _ = inf_mod._openai_llama_admission_reserve( + request = _Request(), llama_backend = inf_mod.get_llama_cpp_backend() + ) + openai_lease = openai_reservation.lease_nowait() + assert openai_lease is not None + assert _snapshot().active == 1 # same key the Anthropic side will use + + task = asyncio.create_task( + anthropic_messages(_payload(), request = _Request(), current_subject = "t") + ) + await asyncio.sleep(0.1) + assert _snapshot().queued == 1 # queued behind the OpenAI generation + openai_lease.release() + assert (await asyncio.wait_for(task, timeout = 2)).status_code == 200 + + asyncio.run(_run()) + + +def test_non_streaming_client_gone_while_queued_returns_499(monkeypatch): + # The disconnect-while-queued branch; nothing else exercised 499. + _install_backend(monkeypatch, slots = 1) + + async def _run(): + held = _occupy(_KEY, 1, 1) + with pytest.raises(HTTPException) as exc: + await anthropic_messages( + _payload(), request = _Request(disconnected = True), current_subject = "t" + ) + assert exc.value.status_code == 499 + assert _snapshot().queued == 0 # waiter cleaned up, not left parked + for lease in held: + lease.release() + + asyncio.run(_run()) + + +def test_streaming_timeout_emits_an_error_event_and_frees_the_slot(monkeypatch): + # Only the non-streaming 503 was covered; streaming reports in-band instead. + monkeypatch.setenv(ADMISSION_QUEUE_TIMEOUT_ENV, "0.15") + monkeypatch.setenv(ADMISSION_KEEPALIVE_INTERVAL_ENV, "0.05") + _install_backend(monkeypatch, slots = 1) + + async def _run(): + held = _occupy(_KEY, 1, 1) # never released, so the waiter times out + response = await anthropic_messages( + _payload(stream = True), request = _Request(), current_subject = "t" + ) + body = await _consume(response) + assert "event: error" in body + assert "message_start" not in body # never reached the model + for lease in held: + lease.release() + assert _snapshot().active == 0 and _snapshot().queued == 0 + + asyncio.run(_run()) + + +def test_fifo_fairness_across_many_waiters(monkeypatch): + _install_backend(monkeypatch, slots = 1) + + async def _run(): + held = _occupy(_KEY, 1, 1) + order = [] + + async def _one(i): + resp = await anthropic_messages(_payload(), request = _Request(), current_subject = "t") + order.append(i) + return resp + + tasks = [asyncio.create_task(_one(i)) for i in range(8)] + await asyncio.sleep(0.2) + assert _snapshot().queued == 8 + held[0].release() + await asyncio.wait_for(asyncio.gather(*tasks), timeout = 5) + assert order == list(range(8)) # granted in arrival order + assert _snapshot().active == 0 and _snapshot().queued == 0 + + asyncio.run(_run()) + + +def test_uncontended_hot_path_is_fast(monkeypatch): + _install_backend(monkeypatch, slots = 4) + + async def _run(): + start = time.perf_counter() + for _ in range(50): + resp = await anthropic_messages(_payload(), request = _Request(), current_subject = "t") + assert resp.status_code == 200 + elapsed = time.perf_counter() - start + # Generous ceiling on purpose: this guards against admission accidentally + # serialising or sleeping on the uncontended path, not against a slow + # runner, so it must not flake on a loaded CI box. + assert elapsed < 10.0, f"50 uncontended round-trips took {elapsed:.2f}s" + assert _snapshot().active == 0 and _snapshot().queued == 0 + + asyncio.run(_run()) + + +async def _drain(body): + chunks = [] + async for chunk in body: + chunks.append(chunk.decode() if isinstance(chunk, (bytes, bytearray)) else chunk) + return "".join(chunks) + + +def test_streaming_midstream_cancel_finalizes_the_monitor(monkeypatch): + # A mid-stream disconnect is delivered as CancelledError so the monitored body + # can finalize its entry. Closing the inner iterator with aclose() instead + # delivers GeneratorExit, and the entry stays "running" for the process life. + _install_backend(monkeypatch, slots = 1) + + async def _run(): + response = await anthropic_messages( + _payload(stream = True), request = _Request(), current_subject = "t" + ) + body = response.body_iterator + await asyncio.wait_for(body.__anext__(), timeout = 2) # stream started + assert inf_mod.api_monitor.active_count() == 1 + + # Propagates back out, as the un-admitted path did; what matters is that + # the monitored body saw it on the way through. + with pytest.raises(asyncio.CancelledError): + await body.athrow(asyncio.CancelledError()) # client vanished + + assert inf_mod.api_monitor.active_count() == 0 + assert _snapshot().active == 0 and _snapshot().queued == 0 + + asyncio.run(_run()) + + +def test_streaming_give_up_while_queued_finalizes_the_monitor(monkeypatch): + # Cancelled before the body ever ran, so nothing downstream can close the + # entry out; the wrapper has to do it. + monkeypatch.setenv(ADMISSION_KEEPALIVE_INTERVAL_ENV, "0.05") + _install_backend(monkeypatch, slots = 1) + + async def _run(): + held = _occupy(_KEY, 1, 1) + response = await anthropic_messages( + _payload(stream = True), request = _Request(), current_subject = "t" + ) + body = response.body_iterator + await asyncio.wait_for(body.__anext__(), timeout = 2) # keep-alive, still queued + assert inf_mod.api_monitor.active_count() == 1 + + await body.aclose() # give up while waiting + + assert inf_mod.api_monitor.active_count() == 0 + for lease in held: + lease.release() + assert _snapshot().active == 0 and _snapshot().queued == 0 + + asyncio.run(_run()) + + +def test_every_dispatch_site_goes_through_admission(): + """All six generation returns in anthropic_messages are admission-wrapped. + + The tool paths need a passthrough-capable backend and a tools payload to reach + at runtime, so guard them structurally instead: a new dispatch site added + without admission (or one reverted to _monitored_anthropic) fails here. + """ + import ast + import inspect + + tree = ast.parse(inspect.getsource(inf_mod).replace("\t", " ")) + handler = next( + node + for node in ast.walk(tree) + if isinstance(node, ast.AsyncFunctionDef) and node.name == "anthropic_messages" + ) + # The wrappers themselves call _monitored_anthropic; only the dispatch sites count. + nested = { + node + for node in ast.walk(handler) + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + and node.name.startswith("_admitted_anthropic") + } + inner = {id(n) for wrapper in nested for n in ast.walk(wrapper)} + + called = [] + for node in ast.walk(handler): + if id(node) in inner or not isinstance(node, ast.Call): + continue + if isinstance(node.func, ast.Name): + called.append(node.func.id) + + assert called.count("_admitted_anthropic") == 6 + assert called.count("_monitored_anthropic") == 0 + + +def test_queued_give_up_runs_the_response_pre_start_cleanup(monkeypatch): + """A stream abandoned while queued must run the builder's eager cleanup. + + The passthrough enters a _TrackedCancel before returning its response and + relies on the stream's finally to exit it. That finally never runs for a + generator that never started, so the response carries a pre-start hook and + the admission wrapper has to chain to it instead of replacing it. + """ + monkeypatch.setenv(ADMISSION_KEEPALIVE_INTERVAL_ENV, "0.05") + _install_backend(monkeypatch, slots = 1) + ran = [] + + async def _hook(): + ran.append(True) + + real = inf_mod._sse_streaming_response + + def _tagged(content, *, unstarted_cleanup = None): + return real(content, unstarted_cleanup = _hook) + + monkeypatch.setattr(inf_mod, "_sse_streaming_response", _tagged) + + async def _run(): + held = _occupy(_KEY, 1, 1) + response = await anthropic_messages( + _payload(stream = True), request = _Request(), current_subject = "t" + ) + body = response.body_iterator + await asyncio.wait_for(body.__anext__(), timeout = 2) # keep-alive, still queued + await body.aclose() # give up before the body ran + + assert ran == [True] + for lease in held: + lease.release() + + asyncio.run(_run()) + + +def test_passthrough_stream_registers_a_pre_start_cleanup(): + # Structural guard: the tracker is entered eagerly, so the response must + # carry the hook that exits it when the body never starts. + import ast + import inspect + + src = inspect.getsource(inf_mod._anthropic_passthrough_stream) + tree = ast.parse(src.replace("\t", " ").lstrip()) + returns = [n for n in ast.walk(tree) if isinstance(n, ast.Return) and n.value is not None] + call = next( + n.value + for n in returns + if isinstance(n.value, ast.Call) + and getattr(n.value.func, "id", "") == "_sse_streaming_response" + ) + hook = next(kw.value for kw in call.keywords if kw.arg == "unstarted_cleanup") + # Not just present: a literal None passes the keyword check and still leaks. + assert isinstance(hook, ast.Call) + assert getattr(hook.func, "id", None) == "_tracked_cancel_unstarted_cleanup" + + +def test_slot_is_released_even_if_closing_the_body_raises(monkeypatch): + # A slot lost here never comes back: with no queue timeout the pool silently + # shrinks and later callers wait forever, so the release must not sit behind + # anything that can throw. + _install_backend(monkeypatch, slots = 1) + + async def _boom(iterator, *, cancelled): + raise RuntimeError("close failed") + + monkeypatch.setattr(inf_mod, "_close_openai_admitted_stream_iterator", _boom) + + async def _run(): + response = await anthropic_messages( + _payload(stream = True), request = _Request(), current_subject = "t" + ) + body = response.body_iterator + await asyncio.wait_for(body.__anext__(), timeout = 2) # stream started + assert _snapshot().active == 1 + + with pytest.raises(RuntimeError): + await body.aclose() + + assert _snapshot().active == 0 # slot returned despite the failure + # And the pool still serves the next caller. + again = get_llama_admission_queue(_KEY).reserve(capacity = 1, config = LlamaAdmissionConfig()) + lease = again.lease_nowait() + assert lease is not None + lease.release() + + asyncio.run(_run()) + + +_CLIENT_TOOLS = [ + {"name": "get_time", "description": "t", "input_schema": {"type": "object", "properties": {}}} +] + + +def _passthrough_payload(**fields): + # server_tools off + declared tools + a passthrough-capable backend routes + # anthropic_messages down the client-tool passthrough dispatch site. + return _payload(tools = _CLIENT_TOOLS, enable_tools = False, **fields) + + +def test_response_pre_start_cleanup_exits_the_passthrough_tracker(monkeypatch): + """A disconnect before the body starts must still exit the cancel tracker. + + The wrapper replaces the response's own pre-start hook, so it has to chain to + it. Asserting through _CANCEL_REGISTRY rather than the wiring, because the + hook can be present and still be a no-op. + """ + backend = _install_backend(monkeypatch, slots = 1) + backend.supports_tool_passthrough = True + monkeypatch.setattr(inf_mod, "_CANCEL_REGISTRY", {}) + + async def _run(): + response = await anthropic_messages( + _passthrough_payload(stream = True), request = _Request(), current_subject = "t" + ) + assert inf_mod._CANCEL_REGISTRY, "passthrough should have registered a tracker" + + cleanup = getattr(response, "_unstarted_cleanup", None) + assert cleanup is not None + await cleanup() # what _SameTaskStreamingResponse runs on a pre-start disconnect + + assert inf_mod._CANCEL_REGISTRY == {} + assert _snapshot().active == 0 and _snapshot().queued == 0 + + asyncio.run(_run()) + + +def test_passthrough_dispatch_site_reserves_and_releases(monkeypatch): + # Behavioural cover for a dispatch site the other tests never reach. + backend = _install_backend(monkeypatch, slots = 1) + backend.supports_tool_passthrough = True + + async def _run(): + held = _occupy(_KEY, 1, 1) + task = asyncio.create_task( + anthropic_messages(_passthrough_payload(), request = _Request(), current_subject = "t") + ) + await asyncio.sleep(0.1) + assert _snapshot().queued == 1 # queued behind the busy slot, not bypassing + for lease in held: + lease.release() + with contextlib.suppress(Exception): + await asyncio.wait_for(task, timeout = 2) # upstream is not mocked + assert _snapshot().active == 0 and _snapshot().queued == 0 + + asyncio.run(_run()) + + +def test_stream_setup_failure_returns_the_slot(monkeypatch): + # count_chat_tokens makes a blocking HTTP call to llama-server, so a dead + # server raises here: after lease_nowait() took the slot, before a body + # exists to release it. Nothing else can hand the slot back. + def _boom(*_a, **_k): + raise RuntimeError("tokenizer unreachable") + + _install_backend(monkeypatch, slots = 1, count_tokens = _boom) + + async def _run(): + with pytest.raises(RuntimeError): + await anthropic_messages(_payload(stream = True), request = _Request(), current_subject = "t") + snap = _snapshot() + assert snap.active == 0, f"slot leaked after stream setup failed: {snap}" + # And the pool still serves the next caller. + again = get_llama_admission_queue(_KEY).reserve(capacity = 1, config = LlamaAdmissionConfig()) + assert again.lease_nowait() is not None + + asyncio.run(_run()) + + +def test_queued_non_stream_cancel_does_not_leak_a_coroutine(monkeypatch): + # The non-stream path builds the generation coroutine before reserving and + # only awaits it once admitted. Giving up while queued must close it. + _install_backend(monkeypatch, slots = 1) + + async def _run(): + held = _occupy(_KEY, 1, 1) + task = asyncio.create_task( + anthropic_messages(_payload(), request = _Request(), current_subject = "t") + ) + await asyncio.sleep(0.1) + assert _snapshot().queued == 1 + task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await task + for lease in held: + lease.release() + + with warnings.catch_warnings(record = True) as caught: + warnings.simplefilter("always") + asyncio.run(_run()) + gc.collect() + leaked = [w for w in caught if "never awaited" in str(w.message)] + assert not leaked, [str(w.message) for w in leaked] + + +def test_stream_timeout_marks_the_monitor_entry_as_error(monkeypatch): + # The finally finishes the entry as "cancelled"; without the fail() first, a + # timed-out request is indistinguishable from a client hang-up in the + # monitor. api_monitor.finish is a no-op on an already terminal entry. + monkeypatch.setenv(ADMISSION_QUEUE_TIMEOUT_ENV, "0.15") + monkeypatch.setenv(ADMISSION_KEEPALIVE_INTERVAL_ENV, "0.05") + _install_backend(monkeypatch, slots = 1) + + async def _run(): + held = _occupy(_KEY, 1, 1) # never released, so the waiter times out + response = await anthropic_messages( + _payload(stream = True), request = _Request(), current_subject = "t" + ) + async for _ in response.body_iterator: + pass + entries = inf_mod.api_monitor.snapshot() + assert entries and entries[0]["status"] == "error", entries + for lease in held: + lease.release() + + asyncio.run(_run()) + + +class _RespawnBackend: + """Backend whose base_url moves to a new port once respawned.""" + + def __init__( + self, + *, + mtp_handled = False, + fallback_in_progress = False, + ): + self.base_url = "http://127.0.0.1:57953" + self.context_length = 4096 + self.respawn_calls = 0 + self._mtp_handled = mtp_handled + self._mtp_runtime_fallback_in_progress = fallback_in_progress + + def count_chat_tokens(self, *_a, **_k): + return 2 + + def _maybe_recover_from_mtp_crash(self, _exc): + return self._mtp_handled + + def _respawn_if_dead(self): + self.respawn_calls += 1 + self.base_url = "http://127.0.0.1:62933" + return True + + +def test_retry_url_stands_down_while_an_mtp_fallback_is_reloading(): + # Only the first caller gets True from _maybe_recover_from_mtp_crash; the rest + # see False and must still stand down, or they respawn the same MTP config + # underneath the fallback already reloading without it. + backend = _RespawnBackend(mtp_handled = False, fallback_in_progress = True) + + url = asyncio.run(_anthropic_passthrough_retry_url(backend, httpx.ConnectError("x"))) + + assert url is None + assert backend.respawn_calls == 0 + + +class _PtRequest: + async def is_disconnected(self): + return False + + +async def _passthrough_response(backend): + return await _anthropic_passthrough_stream( + _PtRequest(), + threading.Event(), + backend, + [{"role": "user", "content": "hi"}], + [], + 0.7, + 0.95, + 20, + 16, + "msg_tracker_probe", + "test-model", + ) + + +def test_disconnect_during_the_opening_lines_exits_the_tracker(): + # Suspended inside emitter.start()'s yields the generator has not reached the + # try/finally that exits the tracker, so those yields need their own. + backend = _RespawnBackend() + + async def _run(): + response = await _passthrough_response(backend) + body = response.body_iterator + await asyncio.wait_for(body.__anext__(), timeout = 2) # first start line + assert inf_mod._CANCEL_REGISTRY, "tracker should be registered" + await body.aclose() + assert inf_mod._CANCEL_REGISTRY == {}, "tracker leaked" + + asyncio.run(_run()) + + +def test_cancel_during_the_opening_lines_exits_the_tracker(): + # Same window, delivered the way _SameTaskStreamingResponse delivers it. + backend = _RespawnBackend() + + async def _run(): + response = await _passthrough_response(backend) + body = response.body_iterator + await asyncio.wait_for(body.__anext__(), timeout = 2) + assert inf_mod._CANCEL_REGISTRY, "tracker should be registered" + with pytest.raises(asyncio.CancelledError): + await body.athrow(asyncio.CancelledError()) + assert inf_mod._CANCEL_REGISTRY == {}, "tracker leaked" + + asyncio.run(_run()) diff --git a/studio/backend/tests/test_anthropic_messages.py b/studio/backend/tests/test_anthropic_messages.py index 621ac9aaca..296cb80911 100644 --- a/studio/backend/tests/test_anthropic_messages.py +++ b/studio/backend/tests/test_anthropic_messages.py @@ -1523,6 +1523,17 @@ def _reset_policy(): reset_tool_policy() +@pytest.fixture(autouse = True) +def _reset_admission_queues(): + # The admission queue is process-global; isolate the shared "llama-server" key + # so one test's leftover reservation can't stall the next. + from core.inference.llama_admission import reset_llama_admission_queues + + reset_llama_admission_queues() + yield + reset_llama_admission_queues() + + class TestAnthropicMessagesToolRouting: class _Request: state = SimpleNamespace() diff --git a/studio/backend/tests/test_anthropic_passthrough_respawn.py b/studio/backend/tests/test_anthropic_passthrough_respawn.py new file mode 100644 index 0000000000..a9f31208ed --- /dev/null +++ b/studio/backend/tests/test_anthropic_passthrough_respawn.py @@ -0,0 +1,262 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. + +"""Restart survival for the Anthropic /v1/messages passthrough. + +A crashed llama-server relaunches on a NEW ephemeral port. Before the retry the +passthrough kept posting to the dead port, so a Claude Code session stayed broken +until the next explicit load. These cover the respawn-and-retry on both the +streaming and non-streaming passthroughs. +""" + +from __future__ import annotations + +import asyncio +import json +import os +import sys +import threading +from types import SimpleNamespace + +import httpx +import pytest + +_backend = os.path.join(os.path.dirname(__file__), "..") +sys.path.insert(0, _backend) + +import routes.inference as inf_mod +from routes.inference import ( + _anthropic_passthrough_non_streaming, + _anthropic_passthrough_retry_url, + _anthropic_passthrough_stream, +) + +_DEAD = "http://127.0.0.1:57953" +_FRESH = "http://127.0.0.1:62933" + + +class _Backend: + """Stub llama backend whose base_url moves to a new port once respawned.""" + + def __init__( + self, + *, + respawn_ok = True, + mtp_handled = False, + ): + self.base_url = _DEAD + self.context_length = 4096 + self.respawn_calls = 0 + self.mtp_calls = 0 + self._respawn_ok = respawn_ok + self._mtp_handled = mtp_handled + + def count_chat_tokens(self, *_args, **_kwargs): + return 2 + + def _maybe_recover_from_mtp_crash(self, _exc): + self.mtp_calls += 1 + return self._mtp_handled + + def _respawn_if_dead(self): + self.respawn_calls += 1 + if not self._respawn_ok: + return False + self.base_url = _FRESH + return True + + +class _Request: + async def is_disconnected(self): + return False + + +class _FakeNonStreamingClient: + def __init__(self): + self.urls = [] + + async def post(self, url, **_kwargs): + self.urls.append(url) + if url.startswith(_DEAD): + raise httpx.ConnectError("connection refused") + return httpx.Response( + 200, + json = { + "choices": [{"message": {"content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 2, "completion_tokens": 1}, + }, + ) + + +def _install_stream_transport(monkeypatch, calls): + def handler(request: httpx.Request) -> httpx.Response: + calls.append(str(request.url)) + if str(request.url).startswith(_DEAD): + raise httpx.ConnectError("connection refused") + content = ( + f"data: {json.dumps({'choices': [{'delta': {'content': 'hi'}}]})}\n\n" + "data: [DONE]\n\n" + ) + return httpx.Response( + 200, + content = content.encode(), + headers = {"content-type": "text/event-stream"}, + ) + + transport = httpx.MockTransport(handler) + real_client = httpx.AsyncClient + + def _client(*_args, **kwargs): + return real_client(transport = transport, timeout = kwargs.get("timeout", 600)) + + monkeypatch.setattr(inf_mod.httpx, "AsyncClient", _client) + + +async def _run_stream(backend): + response = await _anthropic_passthrough_stream( + _Request(), + threading.Event(), + backend, + [{"role": "user", "content": "hi"}], + [], + 0.7, + 0.95, + 20, + 16, + "msg_1", + "test-model", + ) + chunks = [] + async for chunk in response.body_iterator: + chunks.append(chunk.decode() if isinstance(chunk, (bytes, bytearray)) else chunk) + return "".join(chunks) + + +async def _run_non_streaming(backend): + return await _anthropic_passthrough_non_streaming( + backend, + [{"role": "user", "content": "hi"}], + [], + 0.7, + 0.95, + 20, + 16, + "msg_1", + "test-model", + ) + + +# ── Helper ──────────────────────────────────────────────────── + + +def test_retry_url_rebuilds_from_the_respawned_base_url(): + backend = _Backend() + + url = asyncio.run(_anthropic_passthrough_retry_url(backend, httpx.ConnectError("x"))) + + assert url == f"{_FRESH}/v1/chat/completions" + assert backend.respawn_calls == 1 + + +def test_retry_url_is_none_when_nothing_respawned(): + backend = _Backend(respawn_ok = False) + + url = asyncio.run(_anthropic_passthrough_retry_url(backend, httpx.ConnectError("x"))) + + assert url is None + + +def test_retry_url_defers_to_the_mtp_crash_recovery(): + # An MTP+tensor crash schedules its own reload; retrying would race it. + backend = _Backend(mtp_handled = True) + + url = asyncio.run(_anthropic_passthrough_retry_url(backend, httpx.ConnectError("x"))) + + assert url is None + assert backend.respawn_calls == 0 + + +def test_retry_url_tolerates_a_backend_without_respawn_hooks(): + backend = SimpleNamespace(base_url = _DEAD) + + url = asyncio.run(_anthropic_passthrough_retry_url(backend, httpx.ConnectError("x"))) + + assert url is None + + +# ── Non-streaming ───────────────────────────────────────────── + + +def test_non_streaming_retries_against_the_new_port(monkeypatch): + client = _FakeNonStreamingClient() + monkeypatch.setattr(inf_mod, "nonstreaming_client", lambda: client) + backend = _Backend() + + response = asyncio.run(_run_non_streaming(backend)) + + assert response.status_code == 200 + assert backend.respawn_calls == 1 + assert client.urls == [f"{_DEAD}/v1/chat/completions", f"{_FRESH}/v1/chat/completions"] + + +def test_non_streaming_raises_when_the_server_stays_dead(monkeypatch): + client = _FakeNonStreamingClient() + monkeypatch.setattr(inf_mod, "nonstreaming_client", lambda: client) + backend = _Backend(respawn_ok = False) + + with pytest.raises(httpx.ConnectError): + asyncio.run(_run_non_streaming(backend)) + + assert client.urls == [f"{_DEAD}/v1/chat/completions"] # no blind retry + + +def test_non_streaming_does_not_retry_an_mtp_crash(monkeypatch): + client = _FakeNonStreamingClient() + monkeypatch.setattr(inf_mod, "nonstreaming_client", lambda: client) + backend = _Backend(mtp_handled = True) + + with pytest.raises(httpx.ConnectError): + asyncio.run(_run_non_streaming(backend)) + + assert backend.respawn_calls == 0 + + +# ── Streaming ───────────────────────────────────────────────── + + +def test_streaming_retries_against_the_new_port(monkeypatch): + calls = [] + _install_stream_transport(monkeypatch, calls) + backend = _Backend() + + blob = asyncio.run(_run_stream(backend)) + + assert backend.respawn_calls == 1 + assert calls == [f"{_DEAD}/v1/chat/completions", f"{_FRESH}/v1/chat/completions"] + # The retried stream really produced the turn, not just a clean-looking stop. + assert "event: message_start" in blob + assert "event: message_stop" in blob + assert "hi" in blob + + +def test_streaming_emits_an_error_event_when_the_server_stays_dead(monkeypatch): + calls = [] + _install_stream_transport(monkeypatch, calls) + backend = _Backend(respawn_ok = False) + + blob = asyncio.run(_run_stream(backend)) + + assert calls == [f"{_DEAD}/v1/chat/completions"] # no blind retry + assert "event: error" in blob + + +def test_streaming_does_not_retry_an_mtp_crash(monkeypatch): + calls = [] + _install_stream_transport(monkeypatch, calls) + backend = _Backend(mtp_handled = True) + + blob = asyncio.run(_run_stream(backend)) + + assert backend.respawn_calls == 0 + assert calls == [f"{_DEAD}/v1/chat/completions"] + assert "event: error" in blob diff --git a/studio/backend/tests/test_llama_admission.py b/studio/backend/tests/test_llama_admission.py index 2f04e81926..f69eb7c5c9 100644 --- a/studio/backend/tests/test_llama_admission.py +++ b/studio/backend/tests/test_llama_admission.py @@ -16,6 +16,7 @@ from core.inference.llama_admission import ( ADMISSION_CONTROL_ENV, ADMISSION_KEEPALIVE_INTERVAL_ENV, ADMISSION_MAX_QUEUE_ENV, + ADMISSION_QUEUE_PER_SLOT_ENV, ADMISSION_QUEUE_TIMEOUT_ENV, DEFAULT_ADMISSION_KEEPALIVE_INTERVAL_S, DEFAULT_ADMISSION_MAX_QUEUE, @@ -28,8 +29,23 @@ from core.inference.llama_admission import ( ) +_ADMISSION_ENV = ( + ADMISSION_CONTROL_ENV, + ADMISSION_QUEUE_TIMEOUT_ENV, + ADMISSION_KEEPALIVE_INTERVAL_ENV, + ADMISSION_MAX_QUEUE_ENV, + ADMISSION_QUEUE_PER_SLOT_ENV, + *llama_admission._LEGACY_ENV.values(), +) + + @pytest.fixture(autouse = True) -def _reset_queues(): +def _reset_queues(monkeypatch): + # Clear ambient settings for every test, not just the ones that remember to: + # a canonical name set on the machine silently beats the legacy name a test + # is exercising, and the queue registry is process-global. + for name in _ADMISSION_ENV: + monkeypatch.delenv(name, raising = False) reset_llama_admission_queues() yield reset_llama_admission_queues() @@ -41,15 +57,25 @@ def test_admission_config_defaults(monkeypatch): ADMISSION_QUEUE_TIMEOUT_ENV, ADMISSION_KEEPALIVE_INTERVAL_ENV, ADMISSION_MAX_QUEUE_ENV, + ADMISSION_QUEUE_PER_SLOT_ENV, + "UNSLOTH_OPENAI_COMPAT_ADMISSION_CONTROL", + "UNSLOTH_OPENAI_COMPAT_ADMISSION_QUEUE_TIMEOUT", + "UNSLOTH_OPENAI_COMPAT_ADMISSION_KEEPALIVE_INTERVAL", + "UNSLOTH_OPENAI_COMPAT_ADMISSION_MAX_QUEUE", ): monkeypatch.delenv(name, raising = False) config = llama_admission_config_from_env() + # Literals, not the module constants: comparing a default to itself would let + # any future value change through silently. assert config.enabled is True - assert config.queue_timeout_s == DEFAULT_ADMISSION_QUEUE_TIMEOUT_S - assert config.keepalive_interval_s == DEFAULT_ADMISSION_KEEPALIVE_INTERVAL_S - assert config.max_queue == DEFAULT_ADMISSION_MAX_QUEUE + assert config.queue_timeout_s is None # wait forever + assert config.keepalive_interval_s == 5.0 + assert config.max_queue is None # no absolute cap + assert config.queue_per_slot == 16 + assert (DEFAULT_ADMISSION_QUEUE_TIMEOUT_S, DEFAULT_ADMISSION_MAX_QUEUE) == (None, None) + assert DEFAULT_ADMISSION_KEEPALIVE_INTERVAL_S == 5.0 def test_admission_config_env_overrides(monkeypatch): @@ -66,6 +92,25 @@ def test_admission_config_env_overrides(monkeypatch): assert config.max_queue is None +def test_admission_config_honors_legacy_openai_compat_env(monkeypatch): + # The queue is shared with /v1/messages now, but existing OPENAI_COMPAT + # settings must keep working. + monkeypatch.setenv("UNSLOTH_OPENAI_COMPAT_ADMISSION_MAX_QUEUE", "7") + monkeypatch.setenv("UNSLOTH_OPENAI_COMPAT_ADMISSION_CONTROL", "off") + + config = llama_admission_config_from_env() + + assert config.max_queue == 7 + assert config.enabled is False + + +def test_admission_config_prefers_neutral_env_over_legacy(monkeypatch): + monkeypatch.setenv("UNSLOTH_OPENAI_COMPAT_ADMISSION_MAX_QUEUE", "7") + monkeypatch.setenv(ADMISSION_MAX_QUEUE_ENV, "3") + + assert llama_admission_config_from_env().max_queue == 3 + + def test_admission_config_positive_queue_timeout_env(monkeypatch): monkeypatch.setenv(ADMISSION_QUEUE_TIMEOUT_ENV, "600") @@ -106,6 +151,160 @@ def test_fifo_capacity_one_grants_next_waiter_on_release(): asyncio.run(_run()) +def test_pool_hands_out_distinct_slots_and_reuses_them(): + async def _run(): + queue = get_llama_admission_queue("http://llama.test") + config = LlamaAdmissionConfig() + + leases = [queue.reserve(capacity = 3, config = config).lease_nowait() for _ in range(3)] + assert sorted(lease.slot for lease in leases) == [0, 1, 2] # one slot each + snapshot = queue.snapshot() + assert (snapshot.active, snapshot.free, snapshot.capacity) == (3, 0, 3) + + # A freed slot returns to the pool and is handed to the next caller. + freed = leases[1].slot + leases[1].release() + assert queue.snapshot().free == 1 + reused = queue.reserve(capacity = 3, config = config).lease_nowait() + assert reused.slot == freed + + reused.release() + leases[0].release() + leases[2].release() + snapshot = queue.snapshot() + assert (snapshot.active, snapshot.free) == (0, 3) + + asyncio.run(_run()) + + +def test_pool_waiter_is_handed_a_real_slot(): + async def _run(): + queue = get_llama_admission_queue("http://llama.test") + config = LlamaAdmissionConfig() + + held = queue.reserve(capacity = 1, config = config).lease_nowait() + waiting = queue.reserve(capacity = 1, config = config) + assert waiting.lease_nowait() is None + assert queue.snapshot().free == 0 + + held.release() + granted = await waiting.wait(0.1) + assert granted is not None and granted.slot == 0 # the slot just freed + granted.release() + + asyncio.run(_run()) + + +def test_shrinking_capacity_retires_slots_beyond_the_new_pool(): + async def _run(): + queue = get_llama_admission_queue("http://llama.test") + config = LlamaAdmissionConfig() + + leases = [queue.reserve(capacity = 4, config = config).lease_nowait() for _ in range(4)] + assert queue.snapshot().capacity == 4 + + # llama-server reloaded with fewer --parallel slots; in-flight holders keep + # running and their slots retire instead of returning to the smaller pool. + shrunk = queue.reserve(capacity = 2, config = config) + assert shrunk.lease_nowait() is None # all 4 still held, nothing free + for lease in leases: + lease.release() + + granted = await shrunk.wait(0.1) + assert granted is not None and granted.slot < 2 + granted.release() + snapshot = queue.snapshot() + assert (snapshot.capacity, snapshot.active, snapshot.free) == (2, 0, 2) + + asyncio.run(_run()) + + +def test_queue_limit_scales_with_the_serving_slots(): + # The wait line follows --parallel: 16 per slot, floored at 64 so a 1-slot + # backend keeps the depth it had before scaling existed. + config = LlamaAdmissionConfig() + assert config.queue_limit(4) == 64 # --parallel 4 (the default) + assert config.queue_limit(8) == 128 # --parallel 8 + assert config.queue_limit(16) == 256 + assert config.queue_limit(1) == 64 # floor, not 16 + assert config.queue_limit(2) == 64 # floor, not 32 + # An explicit cap wins, and a None multiplier means an unbounded line. + assert LlamaAdmissionConfig(max_queue = 5).queue_limit(8) == 5 + assert LlamaAdmissionConfig(queue_per_slot = None).queue_limit(8) is None + # Non-positive settings mean unbounded, never "reject everything". + assert LlamaAdmissionConfig(max_queue = 0).queue_limit(4) is None + assert LlamaAdmissionConfig(max_queue = -1).queue_limit(4) is None + assert LlamaAdmissionConfig(queue_per_slot = 0).queue_limit(4) is None + assert LlamaAdmissionConfig(queue_per_slot = -3).queue_limit(4) is None + + +def test_queue_limit_rejects_only_once_the_line_is_full(): + async def _run(): + queue = get_llama_admission_queue("http://llama.test") + # Explicit cap, so the test drives rejection without standing up the 64 + # waiters the scaled floor would otherwise require. + config = LlamaAdmissionConfig(max_queue = 4) + + held = [queue.reserve(capacity = 2, config = config).lease_nowait() for _ in range(2)] + parked = [queue.reserve(capacity = 2, config = config) for _ in range(4)] + assert queue.snapshot().queued == 4 + + with pytest.raises(LlamaAdmissionQueueFull): + queue.reserve(capacity = 2, config = config) + + for reservation in parked: + reservation.cancel() + for lease in held: + lease.release() + + asyncio.run(_run()) + + +def test_waiting_is_never_timed_out_by_default(): + # "Wait forever": the default config sets no queue timeout at all. + assert llama_admission_config_from_env().queue_timeout_s is None + assert LlamaAdmissionConfig().queue_timeout_s is None + + +def test_single_request_at_a_time_never_queues_or_allocates_waiters(): + # The common serving case: one request in flight at a time must take a slot + # straight away and never touch the wait line. + async def _run(): + queue = get_llama_admission_queue("http://llama.test") + config = LlamaAdmissionConfig() + for _ in range(50): + reservation = queue.reserve(capacity = 4, config = config) + lease = reservation.lease_nowait() + assert lease is not None # admitted immediately + assert queue.snapshot().queued == 0 # nobody ever lined up + lease.release() + snapshot = queue.snapshot() + assert (snapshot.active, snapshot.free, snapshot.queued) == (0, 4, 0) + + asyncio.run(_run()) + + +def test_unbounded_queue_keeps_waiting_instead_of_rejecting(): + # queue_per_slot None is the "pool + unbounded wait line" mode: nothing is + # ever rejected, callers just line up for the next free slot. + async def _run(): + queue = get_llama_admission_queue("http://llama.test") + config = LlamaAdmissionConfig(max_queue = None, queue_per_slot = None) + + held = queue.reserve(capacity = 1, config = config).lease_nowait() + waiters = [queue.reserve(capacity = 1, config = config) for _ in range(200)] + assert queue.snapshot().queued == 200 # no LlamaAdmissionQueueFull + + held.release() + first = await waiters[0].wait(0.1) + assert first is not None + first.release() + for waiter in waiters[1:]: + waiter.cancel() + + asyncio.run(_run()) + + def test_queue_full_rejects_excess_waiter(): async def _run(): queue = get_llama_admission_queue("http://llama.test") @@ -288,6 +487,105 @@ def test_lease_release_is_idempotent_under_concurrent_calls(): asyncio.run(_run()) +def test_releasing_a_stale_lease_does_not_free_someone_elses_slot(): + # The concurrent test above passes without the _released guard: the racing + # calls all target a still-live slot, which the bitmask already absorbs. The + # case the guard exists for is a slot released twice with a reuse in between. + # It is live: _wait_for_openai_admission_non_streaming releases and re-raises, + # then the caller's finally cancels the reservation and releases the same + # lease again, by which point the slot can belong to another request. + async def _run(): + queue = get_llama_admission_queue("http://llama.test") + config = LlamaAdmissionConfig() + + stale = queue.reserve(capacity = 1, config = config).lease_nowait() + stale.release() + other = queue.reserve(capacity = 1, config = config).lease_nowait() + assert other.slot == stale.slot # the slot got reused + + stale.release() + assert queue.snapshot().active == 1, "stale release handed back a live slot" + other.release() + assert queue.snapshot().active == 0 + + asyncio.run(_run()) + + +def test_grant_reclaims_the_slot_when_the_waiters_loop_is_gone(): + # _grant_waiters_locked takes the slot before scheduling delivery, so if the + # schedule fails the bit is already set. Leaving it set strands the slot for + # good, because _free is rebuilt from the bitmask. + queue = get_llama_admission_queue("http://llama.test") + config = LlamaAdmissionConfig() + held = None + + dead = asyncio.new_event_loop() + try: + + async def _fill_and_queue(): + nonlocal held + held = queue.reserve(capacity = 1, config = config).lease_nowait() + assert queue.reserve(capacity = 1, config = config).lease_nowait() is None + + dead.run_until_complete(_fill_and_queue()) + finally: + dead.close() + + held.release() # grant path now hits the closed loop + assert queue.snapshot().active == 0 + assert queue.is_idle() + + +def test_cancel_returns_the_granted_slot_when_the_waiters_loop_is_gone(): + # Routes cancel() from finally blocks, so a raise here would mask their + # exception and skip the release that hands the granted slot back. + queue = get_llama_admission_queue("http://llama.test") + config = LlamaAdmissionConfig() + held = reservation = None + + dead = asyncio.new_event_loop() + try: + + async def _fill_and_queue(): + nonlocal held, reservation + held = queue.reserve(capacity = 1, config = config).lease_nowait() + reservation = queue.reserve(capacity = 1, config = config) + + dead.run_until_complete(_fill_and_queue()) + held.release() # promotes the waiter, so cancel() has a lease to return + finally: + dead.close() + + reservation.cancel() + assert queue.snapshot().active == 0 + assert queue.is_idle() + + +def test_delivery_to_an_already_finished_waiter_releases_the_slot(): + # A slot is taken before delivery is scheduled, so if the waiter finishes in + # that window someone has to hand it back. _deliver_lease does it twice over, + # in the dead-waiter branch and in the InvalidStateError backstop; this pins + # the outcome, not which one. Reaches into the waiter because no public call + # leaves that window open: queue.cancel() reclaims granted_lease itself. + async def _run(): + queue = get_llama_admission_queue("http://llama.test") + config = LlamaAdmissionConfig() + + held = queue.reserve(capacity = 1, config = config).lease_nowait() + reservation = queue.reserve(capacity = 1, config = config) + waiter = reservation._waiter + + held.release() # schedules _deliver_lease, sets granted_lease + waiter.future.cancel() # finishes the future before the callback runs + assert waiter.granted_lease is not None + await asyncio.sleep(0) # let the callback run + + assert queue.snapshot().active == 0 + assert queue.is_idle() + + asyncio.run(_run()) + + def test_new_key_evicts_idle_prior_load_queues(): # Each model load carries a fresh ephemeral port, so a new base_url key must # not leave the drained queues from earlier loads accumulating forever. @@ -318,3 +616,234 @@ def test_new_key_retains_in_flight_prior_load_queue(): assert set(llama_admission._QUEUES) == {"http://127.0.0.1:2003"} asyncio.run(_run()) + + +def test_capacity_shrink_never_admits_past_the_new_ceiling(): + # A load that downshifts --parallel (or an unload resetting it to 1) shrinks the + # pool while slots are still held. Those holdovers keep occupying the backend, so + # they must count against the ceiling; sizing on free ids alone over-admits. + async def _run(): + queue = get_llama_admission_queue("http://llama.test") + config = LlamaAdmissionConfig() + + held = [queue.reserve(capacity = 4, config = config).lease_nowait() for _ in range(4)] + assert all(lease is not None for lease in held) + waiter = queue.reserve(capacity = 4, config = config) + + queue.reserve(capacity = 1, config = config) # capacity collapses to 1 + # Release the one id that still falls inside the shrunk pool, so it goes + # back on the free list; ids at or above capacity retire instead. + low = min(held, key = lambda lease: lease.slot) + assert low.slot == 0 + low.release() + + # The other 3 holdovers are still generating, which already meets the new + # ceiling, so the freed id must not be handed on. Gating on "is an id free" + # alone grants it here and puts 4 generations on a 1-slot backend. + with pytest.raises(asyncio.TimeoutError): + await waiter.wait(0.2) + assert queue.snapshot().active == 3 + + waiter.cancel() + for lease in held: + if lease is not low: + lease.release() + + asyncio.run(_run()) + + +def test_queue_per_slot_env_is_parsed(monkeypatch): + monkeypatch.setenv(ADMISSION_QUEUE_PER_SLOT_ENV, "4") + assert llama_admission_config_from_env().queue_limit(32) == 128 + # Non-positive asks for an unbounded line rather than rejecting everything. + monkeypatch.setenv(ADMISSION_QUEUE_PER_SLOT_ENV, "0") + assert llama_admission_config_from_env().queue_limit(32) is None + + +def test_max_queue_zero_from_env_is_unbounded_end_to_end(monkeypatch): + # Guards the whole env path, not just the parsed field: a regression that let + # queue_per_slot survive MAX_QUEUE=0 would silently re-bound the line. + monkeypatch.setenv(ADMISSION_MAX_QUEUE_ENV, "0") + config = llama_admission_config_from_env() + assert config.max_queue is None and config.queue_per_slot is None + assert config.queue_limit(1) is None and config.queue_limit(64) is None + + +def test_legacy_env_fallback_covers_every_setting(monkeypatch): + for canonical, legacy in llama_admission._LEGACY_ENV.items(): + monkeypatch.delenv(canonical, raising = False) + monkeypatch.setenv(legacy, "0" if "CONTROL" in canonical else "7") + config = llama_admission_config_from_env() + assert config.enabled is False + assert config.queue_timeout_s == 7.0 + assert config.keepalive_interval_s == 7.0 + assert config.max_queue == 7 + + +def test_empty_canonical_env_falls_through_to_legacy(monkeypatch): + # The branch _raw_env exists for: set but blank must not mask the legacy name. + monkeypatch.setenv(ADMISSION_CONTROL_ENV, " ") + monkeypatch.setenv(llama_admission._LEGACY_ENV[ADMISSION_CONTROL_ENV], "0") + assert llama_admission_config_from_env().enabled is False + + +def test_explicit_queue_per_slot_is_not_floored(monkeypatch): + # The floor exists so a 1-slot backend keeps its old depth by default, not to + # override an operator who asked for a shallow line. + monkeypatch.setenv(ADMISSION_QUEUE_PER_SLOT_ENV, "2") + config = llama_admission_config_from_env() + assert config.queue_limit(1) == 2 + assert config.queue_limit(8) == 16 + + # Unset, the default multiplier is floored instead. + monkeypatch.delenv(ADMISSION_QUEUE_PER_SLOT_ENV, raising = False) + assert llama_admission_config_from_env().queue_limit(1) == 64 + + # A value that does not parse falls back to the default multiplier, so it has + # to keep the default's floor. Otherwise a typo quietly shrinks the line 4x. + for garbage in ("abc", "1e3", "16.0"): + monkeypatch.setenv(ADMISSION_QUEUE_PER_SLOT_ENV, garbage) + assert llama_admission_config_from_env().queue_limit(1) == 64, garbage + + +def test_module_imports_on_python_39(monkeypatch): + """No 3.10+ API on an import path. The package declares >=3.9 but CI only + runs 3.12, so a regression here would ship broken.""" + import ast + import pathlib + + src = pathlib.Path(llama_admission.__file__).read_text(encoding = "utf-8") + tree = ast.parse(src) + + # int.bit_count() (3.10+) + assert not [ + n + for n in ast.walk(tree) + if isinstance(n, ast.Call) + and isinstance(n.func, ast.Attribute) + and n.func.attr == "bit_count" + ] + # dataclass(slots = ...) is 3.10+, so every dataclass must take it through + # the version gate instead of naming it. A new one that forgets the gate + # loses slots silently, so require the **_SLOTS unpack rather than allow it. + seen = 0 + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + name = getattr(node.func, "id", None) or getattr(node.func, "attr", None) + if name != "dataclass": + continue + seen += 1 + assert "slots" not in {kw.arg for kw in node.keywords} + assert [ + kw + for kw in node.keywords + if kw.arg is None and getattr(kw.value, "id", None) == "_SLOTS" + ], ast.dump(node) + assert seen + + +def test_slots_gate_matches_the_running_interpreter(): + """The gate is only worth having if it actually applies where it can.""" + import sys + + gated = (LlamaAdmissionConfig, llama_admission.LlamaAdmissionSnapshot, llama_admission._Waiter) + if sys.version_info >= (3, 10): + assert llama_admission._SLOTS == {"slots": True} + for cls in gated: + assert getattr(cls, "__slots__", None), cls + else: + assert llama_admission._SLOTS == {} + + # Construct through the gate either way: slots=True rebuilds the class, so a + # field it cannot carry over would only show up on instantiation. + config = LlamaAdmissionConfig(max_queue = 7) + assert config.max_queue == 7 and config.queue_limit(4) == 7 + assert llama_admission.LlamaAdmissionSnapshot("k", 1, 1, 0).capacity == 1 + + +def test_held_count_tracks_the_bitmask(): + # _held replaces int.bit_count(); the two must never drift apart. + async def _run(): + queue = get_llama_admission_queue("http://llama.test") + config = LlamaAdmissionConfig() + popcount = lambda: bin(queue._in_use).count("1") + + leases = [queue.reserve(capacity = 4, config = config).lease_nowait() for _ in range(4)] + assert queue._held == popcount() == 4 + leases[1].release() + assert queue._held == popcount() == 3 + shrunk = queue.reserve(capacity = 2, config = config) # shrink with slots held + assert queue._held == popcount() == 3 + shrunk.cancel() # else it is granted a slot as the others drain + for lease in leases: + lease.release() + assert queue._held == popcount() == 0 + + asyncio.run(_run()) + + +def test_snapshot_free_never_exceeds_what_can_be_admitted(): + # After a shrink, low ids can sit in _free while holdovers fill the ceiling. + # Reporting them as free made the admission log contradict itself. + async def _run(): + queue = get_llama_admission_queue("http://llama.test") + config = LlamaAdmissionConfig() + + held = [queue.reserve(capacity = 4, config = config).lease_nowait() for _ in range(4)] + queue.reserve(capacity = 1, config = config) # capacity collapses to 1 + min(held, key = lambda lease: lease.slot).release() + + snapshot = queue.snapshot() + assert snapshot.free == 0, snapshot # nothing is actually takeable + assert snapshot.active == 3 + for lease in held: + lease.release() + + asyncio.run(_run()) + + +def test_a_newcomer_does_not_barge_past_a_parked_waiter(): + # Anti-starvation, pinned as behaviour rather than as the `if not self._waiters` + # check: _take_slot_locked consults _can_admit_locked anyway, so either alone + # refuses the newcomer. This fails if both ever go. + async def _run(): + queue = get_llama_admission_queue("http://llama.test") + config = LlamaAdmissionConfig() + + held = queue.reserve(capacity = 1, config = config).lease_nowait() + parked = queue.reserve(capacity = 1, config = config) + assert parked.lease_nowait() is None + + held.release() + newcomer = queue.reserve(capacity = 1, config = config) + assert newcomer.lease_nowait() is None, "newcomer barged past the parked waiter" + assert (await parked.wait(0.1)) is not None + + asyncio.run(_run()) + + +def test_dead_waiters_stop_counting_against_the_queue_limit(): + # A future cancelled out of band leaves the entry in the deque: cancel() is not + # called, so only the prune drops it. Without that, depth, is_idle() and the + # queue-full limit all drift for the life of the queue. + async def _run(): + queue = get_llama_admission_queue("http://llama.test") + config = LlamaAdmissionConfig(max_queue = 2) + + held = queue.reserve(capacity = 1, config = config).lease_nowait() + first = queue.reserve(capacity = 1, config = config) + second = queue.reserve(capacity = 1, config = config) + assert queue.snapshot().queued == 2 + with pytest.raises(LlamaAdmissionQueueFull): + queue.reserve(capacity = 1, config = config) + + first._waiter.future.cancel() + second._waiter.future.cancel() + assert queue.snapshot().queued == 0, "dead waiters still occupy the line" + # The freed depth is usable again, and an idle queue is evictable. + queue.reserve(capacity = 1, config = config).cancel() + held.release() + assert queue.is_idle() + + asyncio.run(_run()) From 032550df96747b9c67342dd6d3d39992b51f45d0 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 27 Jul 2026 05:01:04 -0700 Subject: [PATCH 040/132] Keep the newer-mapper probe alive when the fetched mapper has no fp8 tables (#7497) * Keep the newer-mapper probe alive when the fetched mapper has no fp8 tables _get_new_mapper reads the two fp8 tables out of the fetched mapper.py under that file's own names, unlike the three NEW_ names it renames itself. A mapper.py that does not define them raises KeyError, the bare except swallows it, and the function returns five empty dicts, so the 4bit and 16bit upgrade check stops firing as well. That check is the reason the probe exists. Every mapper.py older than the fp8 tables is such a file: fetching the 2025-11-07 one leaves the probe with [0, 0, 0, 0, 0] instead of [400, 997, 591]. Reading the two names with .get keeps the 4bit half working and empties only the fp8 half, which costs nothing, since the probe runs only after the installed tables have already missed. Add a regression test that also pins the fetched-only fp8 upgrade error, which the existing test cannot catch: it serves the repo's own mapper.py as both the installed and the fetched source, so any fresh dict satisfies its identity assertions. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- tests/test_new_mapper_fetched_fp8.py | 155 +++++++++++++++++++++++++++ unsloth/models/loader_utils.py | 8 +- 2 files changed, 161 insertions(+), 2 deletions(-) create mode 100644 tests/test_new_mapper_fetched_fp8.py diff --git a/tests/test_new_mapper_fetched_fp8.py b/tests/test_new_mapper_fetched_fp8.py new file mode 100644 index 0000000000..2835aadb59 --- /dev/null +++ b/tests/test_new_mapper_fetched_fp8.py @@ -0,0 +1,155 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. + +"""Regression tests for what ``_get_new_mapper`` hands back to the upgrade probe. + +``test_new_mapper_no_global_leak.py`` serves the repo's own ``mapper.py`` as both installed +and fetched source, so it cannot tell a fetched table from a fresh copy of the installed one. +Two gaps it misses: + +1. The probe must answer for an fp8 repo only the FETCHED mapper knows, so an extra ``"8"`` + entry is spliced into the fetched source only. Isolating the exec without returning the + fetched fp8 tables would silently drop the fp8 half of the upgrade check. +2. The probe must survive a fetched ``mapper.py`` with no fp8 tables (anything older, or a + future rename): reading them with ``[]`` raises ``KeyError`` into the bare ``except``, + taking the 4bit half, the probe's whole purpose, down with it. + +``loader_utils`` imports torch, so ast-extract the resolvers and run them against a stubbed +``requests``, as in ``tests/test_bad_mappings_redirect.py``. +""" + +import ast +import os +import sys +import types + +_MODELS = os.path.join(os.path.dirname(__file__), os.pardir, "unsloth", "models") + +_WANTED = {"__get_model_name", "_resolve_with_mappers", "_get_new_mapper", "get_model_name"} + +# An fp8 ("8") model, spliced into the FETCHED mapper only. +_NEW_KEY = "unsloth/Zeta-9B-Only-On-Main" +_NEW_OFFICIAL = "zeta-org/Zeta-9B-Only-On-Main-FP8" +_NEW_BLOCK = "unsloth/Zeta-9B-Only-On-Main-FP8-Block" +_NEW_ROW = "unsloth/Zeta-9B-Only-On-Main-FP8-Row" +_ANCHOR = ' "unsloth/Kimi-K2-Instruct-BF16" : (' + + +def _mapper_source(): + with open(os.path.join(_MODELS, "mapper.py"), encoding = "utf-8") as f: + return f.read() + + +def _with_extra_fp8_model(source): + assert _ANCHOR in source, "anchor moved; update this test" + entry = ( + f' "{_NEW_KEY}" : {{\n' + f' "16" : ("{_NEW_KEY}", "zeta-org/Zeta-9B-Only-On-Main"),\n' + f' "8" : ("{_NEW_OFFICIAL}", "{_NEW_BLOCK}", "{_NEW_ROW}"),\n' + f" }},\n" + ) + return source.replace(_ANCHOR, entry + _ANCHOR, 1) + + +def _without_fp8_tables(source): + """A mapper.py from before the fp8 tables existed.""" + return source.replace("FLOAT_TO_FP8_BLOCK_MAPPER", "SOME_OTHER_BLOCK_TABLE").replace( + "FLOAT_TO_FP8_ROW_MAPPER", "SOME_OTHER_ROW_TABLE" + ) + + +class _FakeResponse: + def __init__(self, text): + self.text = text + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + +def _install_fake_requests(monkeypatch, text): + module = types.ModuleType("requests") + module.get = lambda url, timeout = None: _FakeResponse(text) + monkeypatch.setitem(sys.modules, "requests", module) + + +def _install_fake_vllm_absent(monkeypatch, namespace): + """vllm >= 0.12.0 returns early from __get_model_name, leaving the probe unreachable.""" + monkeypatch.delitem(sys.modules, "vllm", raising = False) + fake = types.ModuleType("importlib") + fake.util = types.SimpleNamespace(find_spec = lambda name: None) + namespace["importlib"] = fake + + +def _load_resolver(installed_source): + """Stand-in for loader_utils' module globals, built from `installed_source`.""" + from unsloth_zoo.utils import Version + + mapper_ns = {} + exec(compile(installed_source, "mapper.py", "exec"), mapper_ns) + + namespace = { + "INT_TO_FLOAT_MAPPER": mapper_ns["INT_TO_FLOAT_MAPPER"], + "FLOAT_TO_INT_MAPPER": mapper_ns["FLOAT_TO_INT_MAPPER"], + "MAP_TO_UNSLOTH_16bit": mapper_ns["MAP_TO_UNSLOTH_16bit"], + "FLOAT_TO_FP8_BLOCK_MAPPER": mapper_ns["FLOAT_TO_FP8_BLOCK_MAPPER"], + "FLOAT_TO_FP8_ROW_MAPPER": mapper_ns["FLOAT_TO_FP8_ROW_MAPPER"], + "SUPPORTS_FOURBIT": True, + "transformers_version": Version("4.57.6"), + "Version": Version, + "os": os, + } + with open(os.path.join(_MODELS, "loader_utils.py"), encoding = "utf-8") as f: + tree = ast.parse(f.read()) + for node in tree.body: + if isinstance(node, ast.Assign) and any( + getattr(t, "id", None) in ("BAD_MAPPINGS", "_OFFLINE_ENV_VALUES", "_OFFLINE_ENV_KEYS") + for t in node.targets + ): + exec(compile(ast.Module([node], []), "", "exec"), namespace) + elif isinstance(node, ast.FunctionDef) and ( + node.name in _WANTED or node.name == "_env_says_offline" + ): + exec(compile(ast.Module([node], []), node.name, "exec"), namespace) + return namespace + + +def test_probe_answers_for_an_fp8_repo_only_the_fetched_mapper_knows(monkeypatch): + installed = _mapper_source() + namespace = _load_resolver(installed) + installed_block = namespace["FLOAT_TO_FP8_BLOCK_MAPPER"] + installed_row = namespace["FLOAT_TO_FP8_ROW_MAPPER"] + assert _NEW_OFFICIAL.lower() not in installed_block, "the installed table must not know it" + + _install_fake_requests(monkeypatch, _with_extra_fp8_model(installed)) + _install_fake_vllm_absent(monkeypatch, namespace) + + try: + resolved = namespace["get_model_name"]( + _NEW_OFFICIAL, load_in_4bit = False, load_in_fp8 = "block" + ) + except NotImplementedError as error: + assert "not supported in your current Unsloth version" in str(error) + else: + raise AssertionError( + f"a fetched-only fp8 repo must raise the upgrade error, got {resolved!r}" + ) + + # Answering must not have adopted the fetched tables. + assert namespace["FLOAT_TO_FP8_BLOCK_MAPPER"] is installed_block + assert namespace["FLOAT_TO_FP8_ROW_MAPPER"] is installed_row + assert _NEW_OFFICIAL.lower() not in namespace["FLOAT_TO_FP8_BLOCK_MAPPER"] + + +def test_probe_survives_a_fetched_mapper_without_the_fp8_tables(monkeypatch): + installed = _mapper_source() + namespace = _load_resolver(installed) + _install_fake_requests(monkeypatch, _without_fp8_tables(installed)) + + int_to_float, float_to_int, map_to_16bit = namespace["_get_new_mapper"]()[:3] + + assert ( + int_to_float and float_to_int and map_to_16bit + ), "a fetched mapper.py without the fp8 tables must not take the 4bit upgrade check down" diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py index 8214adc0bf..7aae75fe4f 100644 --- a/unsloth/models/loader_utils.py +++ b/unsloth/models/loader_utils.py @@ -205,8 +205,12 @@ def _get_new_mapper(): namespace["NEW_INT_TO_FLOAT_MAPPER"], namespace["NEW_FLOAT_TO_INT_MAPPER"], namespace["NEW_MAP_TO_UNSLOTH_16bit"], - namespace["FLOAT_TO_FP8_BLOCK_MAPPER"], - namespace["FLOAT_TO_FP8_ROW_MAPPER"], + # .get, not []: these two come from the fetched file under its own names (unlike + # the NEW_ names above, renamed here), so an older or renamed mapper.py would + # KeyError into the bare except and take the 4bit half of the probe down too. + # {} is safe: the probe runs only after the installed tables already missed. + namespace.get("FLOAT_TO_FP8_BLOCK_MAPPER", {}), + namespace.get("FLOAT_TO_FP8_ROW_MAPPER", {}), ) except: return {}, {}, {}, {}, {} From da447d47ba725c2519ae494aea57834f16d4ad62 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 27 Jul 2026 05:02:06 -0700 Subject: [PATCH 041/132] Studio: fix the "No model loaded" error, and optionally auto-download a model named in an API request (#7454) * Studio: say which model is missing instead of "No model loaded" A /v1 request naming a model that is not downloaded returned the generic "No model loaded. Call POST /inference/load first.", which cannot fix it. Return 404 model_not_found naming the model and listing what can serve, and make the API usage examples name a model the server actually has. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Studio: page the API monitor, show model load/unload, pin the example quant The monitor rendered all 50 retained entries in one scroller: page it 5 at a time, freezing history while paged back so live traffic cannot reorder it. Add model load/unload rows so the feed shows what the server is doing, and stop the header rendering the loaded model as a raw host path. Advertise each model's GGUF quant on /v1/models so the example pins repo:QUANT, and move the auto-switch section above the monitor with shorter copy. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Studio: optionally download a model named in an OpenAI API request Auto-switch only ever loaded models already on disk, so naming one this server does not have either 404s or, when something else is loaded, gets quietly answered by the resident model. Add openai_api_auto_download_model (off by default, gated on auto-switch). When on, a /v1 request naming a GGUF repo that is not downloaded starts a background fetch and returns 503 with Retry-After and a typed model_downloading code. The resident model keeps serving in the meantime, and the retry after the download completes is served by the new model through the existing auto-switch path. The download reuses the Hub manager's service layer, which already does repo-id validation, casing, claim bookkeeping, disk preflight, resume and cancel. The in-loader download is deliberately not used: it silently falls back to a smaller quant under low disk, which is wrong when the caller named an exact one. Admission is narrow, since a request only needs an API key: - namespace/name only, so gpt-4 and other foreign ids fall through to the resident model exactly as before - GGUF only, decided from the remote file list rather than the repo name - anything declaring auto_map is refused, so trust_remote_code stays a deliberate opt-in in the UI and can never be granted over the API - a single download at a time, plus a free-disk reserve - one model_info call answers existence, gating and the quant list, so a missing repo, a gated repo and a wrong quant each get their own error With the setting off every one of these paths is byte-identical to before. Also: - monitor rows for downloads, with a live percentage - public_model_id resolves an HF cache snapshot to its repo id, so a cache-loaded model is no longer labelled with a commit sha; this drops the duplicate helper added for the monitor and fixes the same leak in the inference status response - the unedited sk-unsloth-YOUR_KEY from the copyable examples now says so instead of "Invalid or expired API key"; every other bad key keeps the generic message * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Studio: add an Unload button to the API monitor The monitor names the loaded model but offered no way to free it. Idle auto-unload is the only existing release path, and it needs a TTL and a wait. The button sits next to Refresh, appears only while a model is loaded and is disabled mid-unload. /unload matches on the internal identifier, which this response deliberately omits because it would be a host path, so the click reads it from /api/inference/status the same way the chat runtime does rather than widening the monitor payload. Also stamp the manual unload row with the quant, read before the teardown clears it, so it reads repo:QUANT like the load row it pairs with. * Studio: keep the API monitor Unload button visible when idle It only rendered while a model was loaded, which hid the one manual release path at exactly the moment someone goes looking for it. Render it always, disabled with a "No model is loaded" tooltip when there is nothing to free. * Studio: never answer a named model with a different one Asking for a model this server is not serving returned 200 from whatever was resident. Requesting gemma-4-E2B-it-GGUF:UD-Q6_K_XL while UD-Q4_K_XL was loaded got a confident answer from the wrong quant, with nothing in the response saying so. A name carrying a namespace (org/model, optionally :QUANT) is a concrete reference, so 404 instead, with the reason: - wrong quant -> names the quants that are actually downloaded - not on disk -> lists what is available - on disk but auto-switch off -> says to turn it on Ids without a namespace (gpt-4, claude-3, default) are foreign labels rather than references, so they still fall through to the resident model and drop-in clients are unaffected. A bare org/model is still satisfied by any loaded quant of that repo; only an explicit :QUANT must match. The check runs whatever the auto-switch and auto-download toggles are, since serving the wrong weights is wrong in every configuration. It is skipped when nothing is loaded, where the existing no-model-loaded error already says the right thing, and when the model is on disk with auto-switch on, where a failed swap should still fall back. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Studio: use a simpler prompt in the API usage examples "What is Unsloth Studio?" rather than "Can Unsloth Studio do API calling?". One constant feeds all nine snippet tabs. * Studio: only refuse a model reference meant for this server A namespace alone was treated as a concrete model reference, so a /v1 request naming anthropic/claude-3.5-sonnet, openai/gpt-4o or any other LiteLLM or OpenRouter style vendor/model id started returning 404 instead of being answered by the resident model. Refuse only on evidence the caller meant this server: an explicit GGUF quant label, or a repo that is actually on disk here. gpt-4 and vendor/model alike fall through again, while the wrong-quant and wrong-repo cases this PR exists for still refuse. Also from review: - Release the single download slot by object identity, not repo id. A stale watcher could clear a newer download of the same repo and let a second multi-GB fetch start alongside it. - Catch BaseException around admission: CancelledError is not an Exception, so a cancelled request stranded the slot for the process lifetime. - Honour the download service's accepted=False, which it returns without raising for a cross-variant conflict, instead of promising a download that was never dispatched. - Treat a failed status probe as unknown rather than idle, so a transient read cannot fail the monitor row and free the slot under a live worker. - Check gated repos with auth_check. The Hub serves metadata for a gated repo without granting its files, so the licence gate was being reported as the unrelated custom-code refusal. - Size the disk reserve from the download plan, which includes the mmproj and MTP companions the worker fetches with every quant. - Never fetch under the server's own HF token. The repo is named by whoever holds an API key, so the ambient token let that key pull the owner's private repos. - Refuse an explicit quant on a backend with no quant identity, gated on the suffix really being a quant so Ollama style :latest tags still match. - Raise instead of falling through when the diagnosis fails: the mismatch is already established by then, only the wording is uncertain. - Report a failed switch as 503 model_switch_failed rather than answering as the resident model. - Fail an open monitor row under the same lock as the check, so a finish landing in between cannot stamp an error onto a completed row. - Usage examples never emit a hardcoded model id: the catalog is tri-state and the panel asks for a model to be loaded instead of printing one the server cannot serve. It also refreshes when the loaded model changes. - Keep the monitor pager reachable while frozen entries expire. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Studio: scope the auto-download 404 cache to the caller's credentials The Hub answers 404 for a private repo the caller cannot see, so caching that verdict per repo alone let one anonymous request mark a private repo unservable for everyone for the whole TTL. A later caller sending a valid X-Unsloth-HF-Token skipped the probe and fell through to the resident model instead of downloading what it asked for. Keyed on the repo id plus a digest of the token now, so the token itself is never held. Two more from the same review: - Clear the chat runtime checkpoint after unloading from the API monitor, as the chat eject flow already does. The store went on treating the freed checkpoint as loaded and the usage examples kept naming it. - Point gated and not-found callers at the X-Unsloth-HF-Token header. Automatic download deliberately ignores the server's own Hugging Face identity, so telling the user to add a token in Studio sent them round the same 403 forever. * Studio: tighten the comments added by this branch * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Studio: keep API auto-download off the server's Hugging Face identity Passing None for the caller's token was not anonymous. spawn_worker substitutes the backend's HF_TOKEN for a falsy one, and HfApi(token=None) falls back to a cached login, so a repo named by an API-key holder could still be fetched under the owner's Hub identity and land in the shared catalog. The metadata probe and auth_check now pass an explicit False, and dispatch threads allow_ambient_token=False so the worker stays anonymous too. The flag defaults to True, so the UI download path keeps the ambient fallback that private repos rely on. Three more from the same review: - Require an exact hf_variant match only when the suffix is really a quant. The llama.cpp branch still compared Ollama style :latest and :8b against the loaded quant and refused the resident model, which is the opposite of what looks_like_quant classifies them as. - Decode an HF cache repo id only when the models-- component is followed by snapshots. An ordinary directory whose name merely starts with models-- was being read as an encoded repo id. - Return the probing response before consulting the job registry when an adopted claim has no variant yet. A stale error on the whole-repo key could otherwise release the slot the first request's probe still holds, letting a second large download start beside it. * Studio: stop treating a namespace as what decides model intent The rule refused a reference only when it carried a namespace, which was wrong in both directions. vendor/model is how LiteLLM and OpenRouter name every provider, and a standalone or custom-folder GGUF is advertised without one, so asking for a path-free local id such as model-Q4_K_M was answered by whatever else happened to be resident. The slashless early return is gone and the same evidence test now applies to every id: an explicit quant, or a model that actually resolves here. gpt-4 and default still fall through because they are not local, not because of their shape. Also: - Recognise bits-per-weight quant labels. _extract_quant_label emits IQ4_XS-3.53bpw and the resolver and downloader both accept it, but _GGUF_KNOWN_QUANT_RE has no bpw group, so looks_like_quant rejected a reference the rest of the machinery understands. - Upper-case the synthetic names handed to _pick_best_gguf. Its preference tokens are upper case and matched case-sensitively, so a repo with lower-case filenames skipped the preference and took the first entry, which can be F16. - Only offer a downloaded but unloaded model as a runnable example when auto-switch is on. It is off by default, so the copied snippet hit the no-model-loaded error, which is the failure this branch exists to fix. The tool-passthrough cancel test stubbed asyncio.to_thread module-wide, so it cancelled at the first thread hop rather than the generation hop it means to test. Model resolution runs off the loop before the monitor row opens, so that stub now passes the resolver through. * Studio: tighten the comments added since the last pass * Studio: match a resident model through its resolver alias A manual load stores the model by its on-disk path while the resolver and /v1/models advertise it as publisher/model, so _loaded_satisfies could not recognise the alias. Reducing the resolution to a boolean then threw away the load path that would have proved the match, and the request was refused with 404 for a model the server was serving at that moment. Common for LM Studio models and custom-folder aliases. The resolved path is compared against the resident backend before anything is refused. Also: - Size disk admission on what is left to fetch. expected_bytes is the whole plan, so a resumed quant or a companion already pulled in by another quant was charged for twice and could 507 a download that fits. Cached blobs are subtracted through existing_blob_bytes, the same accounting the worker's own preflight does, and it falls open to the full size when no blob hashes are available. - Report a cancelled download as cancelled. The catch-all sent every state other than complete or idle through fail_open, so a deliberate cancel rendered as a download failure rather than the monitor's cancelled state. - Keep polling the servable ids while nothing is loaded. The poll settled as soon as auto-switch was on, so turning it back off left the examples naming an unloaded model until something else remounted the panel. * Studio: shorten the comments added in the last pass * Studio: keep the FLA fast-path tests hermetic across transformers versions _discover_fla_model_types scans the *installed* transformers for modeling files importing `from fla.`, so `models/qwen3_5/` only exists from transformers 5.x. The backend supports transformers>=4.51, and on a 4.x install the Qwen3.5 gate returns False, so 14 tests in test_training_worker_flash_attn.py silently exercised a no-op instead of the install path and failed their call-count assertions. Pin the discovered model_type set in those 14 tests, the same way test_hook_does_not_install_tilelang_for_model_outside_allowlist already pins it against newly added FLA model_types. Test-only change: the production gate and the _discover_fla_model_types unit tests are untouched. * Studio: keep the /v1 admission check off the model-scanning path The admission check added here runs on every /v1 request, including with auto-switch off, where the route used to return straight away. It called resolve_local_gguf, whose index is cached for 5s and otherwise rebuilt by walking ./models and every HF cache root, under a lock the next caller waits on. On an install with a large cache that scan measured 6.1s, longer than the TTL that is meant to amortise it, so steady traffic would keep rebuilding it. Answer from the last built index instead and never rebuild from the request path: a stale answer is fine here, since what is on disk barely moves and a finished download already invalidates the index. The first request, before any scan has completed, warms the index on a background thread and skips the check rather than blocking on it. That also makes the lookup a dict read, so it no longer needs handing to a thread. Cold resolve on this box goes from 6152495us to 0.4us, and the whole hook now costs the same for a foreign label as for the resident model. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Studio: fix the admission hook's cold, stale and contended index paths Five review items, four of them on the admission hook added here. Skipping the check until the first scan lands also skipped explicit quant mismatches, so the first request after startup could ask for :Q8_0 while Q4_K_M was resident and be answered by it. The early return was redundant as well: with an empty index resolved is None and here is False, so the gate below already lets a bare name through and refuses an explicit quant, which is what the except branch has always concluded. Dropped it and index_is_built with it. index_is_built took _lock, which _index holds for the whole scan, so once a warm was running every later request blocked on the event loop for exactly as long as the scan it was there to avoid. The warm now has its own lock and reads the timestamp unlocked, which is safe because _scan is only ever rebound. Warming only when the index had never been built left a model fetched in the Hub UI, or dropped into a scan folder, invisible for the life of the process, since only the auto-download watcher calls invalidate_index. Warm on staleness too, and unconditionally, so it refreshes within a TTL without a scan on the request path. Rescanning is capped at a tenth of the scan's own duration: a big install takes longer to scan than the TTL, and warming on the TTL alone would keep a thread scanning continuously. An Ollama-style tag names no quant, so the resolver misses it and auto-download saw a model the resident one already answers to, then 404'd it for having no such quant. Return early when the loaded model satisfies the reference. Frontend: a cancelled download said "Model download failed", because the label collapsed everything non-completed into failure. The backend tests get an autouse fixture that stops the warm from walking the developer's real HF caches; that scan starved the loop under the timing sensitive streaming tests. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Studio: make /v1/models and the admission hook agree on what is local Three review items, all on the seam between the catalog scan and the resolver index, which run on separate schedules. /v1/models can advertise a local GGUF the resolver has not indexed yet. A bare id carries no quant to refuse on, so a client asking for one it had just been handed was answered by the resident model instead. The hook now reads the catalog cache as evidence too, never scanning it. It takes the path rather than a yes/no because the converse also happens: the catalog can list the resident weights under an alias the loaded entry does not answer to, and those must stay served. That alias was also emitted twice by /v1/models, once as the loaded basename a manual load records and once as publisher/model marked unloaded, because the dedup only compared ids. Compare the path as well. A directly loaded standalone .gguf takes its quant from the filename, but the resolver stores such files with no quants, so the advertised : stopped resolving as soon as anything else loaded. Advertise a quant only when that reference resolves, and downgrade only on a definite answer so a cold index leaves the metadata alone. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Studio: tighten the comments this branch adds Collapse the multi-line notes in the auto-download path, the /v1 admission hook and their tests to one line each, keeping the reason and dropping the restatement. No behaviour change. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Studio: four admission and catalog fixes from review Lowercasing paths in _resolves_to_resident made /srv/models/Foo and /srv/models/foo the same weights on any case-sensitive filesystem, so a request for one could be answered by the other and /v1/models could mark the wrong entry loaded. That helper now backs residency as well as admission, so use os.path.normcase, which folds case only where the filesystem does. Advertising a quant whenever the resolver could not disprove it kept the bug it was meant to fix: a standalone .gguf loaded before the first scan still got : published, and the usage examples persist that. No proof is not proof, so omit it and warm the index instead. A 401 from an expired or invalid X-Unsloth-HF-Token skipped the 403 and 404 branches and surfaced as "could not reach Hugging Face, retry shortly". It now says to replace the token, kept apart from the gated refusal since a rejected credential is not an unaccepted licence. An image request naming an undownloaded text-only GGUF started the whole download and only then hit the capability guard, which never sees a remote target, so every retry 400d and the bytes were wasted. Thread require_vision into admission and check it against the mmproj companions the disk preflight already asks build_gguf_variant_plans for. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Studio: make the Hub error fixture carry a status on both hub majors The 401 test built HfHubHTTPError directly, which works on 0.x and fails on 1.x where response is required and keyword-only, so all four Python jobs failed while the same test passed locally. _hub_error already handled both constructors, but the 0.x branch left the exception with no response at all, and hf_error_status reads the status off it for the types that do not encode it in their name. So it could only produce a usable error on 1.x, which is why the test bypassed it. Attach the status when the constructed exception lacks it, and use the helper. Cover the helper itself against stand-ins for both constructor shapes, since whichever hub is installed only ever exercises one of them. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Studio: invalidate on every download, resolve bare tags, keep polling Three review items. Only the API auto-download watcher dropped the resolver cache, so a GGUF fetched in the Hub UI stayed absent to the cache-only request path and the request was answered by whatever was resident. finalize_worker_exit is the one point every download worker exits through, so invalidate there. That closes the window without leaning on the TTL, which the scan-duration throttle can stretch past 5s on an install where the scan itself takes longer than that. A downloaded but unloaded GGUF asked for as org/model:latest missed the resolver, since the suffix was always treated as an exact quant. With auto-download on that probed the Hub and returned a 404 for a quant that was never a quant; with it off it refused without switching. Fall back to the base entry when the suffix is not quant-shaped, and keep exact matching for real quants so a swap can never serve the wrong weights under the right name. The usage examples stopped polling once a model was resident, but idle unload frees one without touching the store, so nothing re-ran the effect and the examples kept naming a model that could no longer be reloaded. Slow the poll to 60s instead of stopping it. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Studio: hold the download slot while it is in use, and keep quants to llama.cpp _loaded_satisfies refuses a quant reference against the Transformers backend by name, but the path match did not carry that rule. A Transformers model active from a directory that also holds GGUF exports therefore matched a request for one of those quants and answered it with the safetensors weights. Only llama.cpp has a quant identity, so admission now passes llama_only whenever the reference is quant-qualified. A bare name still matches either backend, and /v1/models residency keeps the default so a loaded Transformers model is still reported loaded. The 24 hour watch window was bounding ownership of the single-flight slot when it should only have been bounding progress reporting, so a legitimately slow download had its slot handed back while the worker was still writing, admitting a second multi-gigabyte download beside it. Resolve the row on the clock, but keep the slot on a slower poll until the job is actually terminal. Past the deadline an unknown state does release it, since it means the worker cannot be probed and holding it on that forever would wedge auto-download. * Studio: keep what the resolver already knew when a download lands Invalidating cleared the index to empty. The request path reads that cache without scanning, so from a completed download until the rebuild landed it had no evidence about any local model, not just the new one, and a bare request for any of them was answered by whatever was resident. Wiring the hook into the shared completion path in the last commit widened that from auto-download to every download. Mark the scan stale and keep the entries instead. Both _index and warm_index_soon rebuild on a zero stamp, while the request path still sees everything it knew a moment ago. Only a completed download invalidates, and that only ever adds models, so nothing retained goes false. Warm from the completion hook too, so the rebuild starts when the download lands rather than when the next request happens to need it. * Studio: match the quant, not just the directory, and default-select bare tags Two quants of one repo share a directory, so the path match could not tell them apart and an explicit :Q8_0 was answered by a resident Q4_K_M that _loaded_satisfies had already refused by name. The llama_only fix in the last commit only ruled out the wrong backend, not the wrong quant on the right one. Both path matches now require the resident hf_variant to equal the requested quant whenever the reference is quantified; a bare name still matches on the path alone, since it claims nothing about the weights. The local resolver already treated a tag that names no quant as meaning the repo, but remote admission still looked for a quant literally called "latest", so the same reference resolved locally and 404d remotely. Branch on looks_like_quant there too. A real quant the repo does not have is still a 404 and never a substitution, which is what separates this from the loader's low-disk fallback. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Studio: one quant preference, and stop trusting a stale checkpoint list_local_gguf_variants sorts by descending size, so the head of variants was the biggest quant, often F16, while remote admission and a plain load both rank through _pick_best_gguf. A bare id therefore meant a different quant depending on which side answered it, and the local answer was the one that could evict a working model and then fail or OOM starting an F16 next to a usable Q4. /v1/models advertised that same head for pinning. Pull the ranking into one preferred_quant helper and have both sides use it. The usage examples returned a stored checkpoint without ever consulting /v1/models, and the polling added last round was gated on not having one, so for a stored checkpoint it never ran. An idle unload then left the panel showing a snippet that could not run. Poll whenever mounted, and prefer the checkpoint only while the catalog still backs it or switching can reload it. A catalog that has not answered yet is not evidence against it. The static contract pinned the old dependency array, so it now asserts the intent it documents: a finished load re-runs the fetch, and the effect is not gated on having no checkpoint. * Studio: fix the Windows path compare, and advertise a label the worker knows The case fix normalized the separator to "/" and then called os.path.normcase, which on Windows folds case and rewrites the separator back to a backslash, so the descendant checks compared against a "/" the path no longer had. A manually loaded GGUF reached through an alias then read as a different model, giving a false 404 and an alias marked unloaded. Run normcase first and normalize the separator after it. There are two quant-label extractors and they only agree while a recognized quant token is present. With none, _extract_quant_label takes the last hyphenated segment, "7b" of llama-7b, while build_gguf_variant_plans and the worker key the whole stem: the plan lookup missed and the job exited on a variant it had no shards for. Use the canonical extractor for the unrecognized case only. Checked across real filenames first, the two match on every recognized quant and part on bpw-qualified labels, which _extract_quant_label keeps apart on purpose so byteshape's IQ4_XS at 3.53, 3.97 and 4.19 bpw stay separate variants. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Studio: a stored checkpoint needs catalog evidence, not just the switch setting Preferring it whenever switching was on short-circuited the catalog check, so a checkpoint the store still held after the model was deleted or moved kept being named even though /v1/models had already proved it absent, and the snippets 404d instead of falling back to a model that is actually there. A lookup rather than a disjunction, which settles the whole matrix in one place: no answer yet keeps the checkpoint, since that is not evidence against it; listed and resident keeps it; listed but unloaded keeps it only when switching can reload it; absent falls back whatever the setting says. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Studio: normalize the quote style pre-commit would have rewritten * Studio: cover the model that just landed, and pin the quant the catalog has Retaining the index on invalidation protects what was already scanned and by construction cannot contain the model that just finished downloading, so a bare request for it in the window before the rebuild was still answered by the resident model. Record the repo at the completion hook and treat that as admission evidence alongside the resolver and the catalog; the next completed scan clears the notes, since the index then covers them. Publishing a rebuilt index before completion becomes observable would have closed it too, but that blocks the download worker for the length of the scan. Catalog membership proves the repo, not the saved quant, and the examples then pinned the stored one. A quant deleted while another quant of the same repo remained produced repo:deleted-quant, a missing-quant 404 with a runnable alternative listed right beside it. Pin what the catalog advertises: for a resident entry that is the resident quant, for an unloaded one it is a quant actually on disk. The store is only consulted before /v1/models has answered. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Studio: apply three rules everywhere they belong, not only where reported The trust probe was the last credential handoff still passing a raw token. huggingface_hub reads None as "use the cached login", so a caller-named repo was read with this server's Hugging Face identity whenever the caller sent none, which is exactly the isolation the metadata probe and the worker already keep. It takes _hub_token now. Enumerated the rest of that path while there: auth_check, model_info and spawn_worker were already correct. finalize_worker_exit is shared with dataset downloads, so the resolver hook fired for every completed dataset, scanning the model directories for nothing and recording the dataset id as local-model evidence, which turns a bare /v1 request naming that id into a refusal instead of a foreign-id fallthrough. Gated on repo_type. _already_serving decided "bare" on the presence of a colon while _loaded_satisfies and the resolver decide it on whether the suffix names a quant, so org/model:latest against a serving Q8_0 read as a mismatch and swapped in the preferred Q4_K_M for a request either one answers. That rule now lives in four places, each fixed in its own round, so this time I looked for the rest and found a fifth: describe_local_miss splits on the bare colon and its docstring claims it splits like the resolver. It no longer did, and would report a missing quant named "latest". Fixed here too, unreported. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Studio: probe before refusing busy, and scan once when the index is cold The busy refusal fired before anything established the requested label was a model at all, so any namespaced id a drop-in client sends was told to wait out an unrelated download for as long as it ran. Probe first and refuse only a label the Hub actually serves as GGUF; anything else falls through to the resident model as before. A probe failure answers "not downloadable", since stranding ordinary traffic costs more than missing a busy refusal. Treating an unbuilt index as "nothing here" let the first request after startup be answered by the resident model under another model's name. That was a deliberate trade to keep the scan off the request path, and it was the wrong one. Cold, the scan now runs once on a thread, bounded so a pathological install falls through rather than hanging the request. Built, the request path still never scans, so the latency fix stands. The watcher freed the slot the moment it saw an error, while Retry-After is thirty times the poll interval, so the client came back to an empty slot and restarted the same failing download instead of being told. Hold the failure on the slot until a retry surfaces it, and let another repo take it after three retry intervals so a client that never returns cannot keep it. The watcher also invalidated on completion, which now lands after finalize_worker_exit's warm and marks that fresh scan stale, pushing a synchronous rescan onto the retry. Removed. _loaded_satisfies lowercased paths as well as aliases, so it returned satisfied before the case-preserving compare below could run. Both now go through one helper: paths compare with normcase, aliases stay case-insensitive. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Studio: an unfinished scan is not absence, and a decided refusal is not a failure Bounding the cold scan then reading the bound as "not here" left the same hole one branch over. A timeout now answers 503 model_indexing with a Retry-After and leaves the warm running. A foreign label sent inside that window is asked to retry rather than falling through, which is a real cost, but the window is one request on an install whose scan exceeds ten seconds and it clears itself, where answering with the wrong weights does not. That uncovered a worse one. Every check here runs inside a broad except whose job is "could not verify, so fall through", so an HTTPException raised in the block was logged as a verification failure and the request was answered by the resident model. Any refusal decided in there was being swallowed. Re-raise it ahead of that handler. Canonicalizing generic labels made them real variant keys, but the matcher still decided on shape, so repo:llama-13b fell past an exact match and fetched llama-7b. Match exactly first, whatever the shape; a quant-shaped suffix that matches nothing is still a miss and never a swap. Marking a catalog alias loaded while publishing the preferred on-disk quant claimed alias:Q4 was loaded while Q8 was serving, and requiring the resident quant to match then made pinning it a 404. Advertise the resident variant when the entry resolves to the resident model. * Studio: keep the asyncio.timeout fallback tests runnable on Python 3.10 Both tests deleted asyncio.timeout to force _wall_clock_timeout down its pre-3.11 branch, but monkeypatch.delattr raises when the attribute is already absent. On Python 3.10, the one version the fallback exists for, there is nothing to delete, so the two tests errored with AttributeError before reaching the code they cover. Passing raising=False makes the deletion a no-op there and leaves the assertions running against the same branch on every version. Every other delattr in the repo already passes raising=False for exactly this reason. Verified with asyncio.timeout removed from the interpreter: the two tests fail with the CI AttributeError before this change and pass after, and the file still runs 89 passed on 3.13 where the deletion is real. * Studio: decide GGUF residency, servability and variant keys by one rule each Four admission and catalog fixes, each closing a gap between two places that were answering the same question differently. The /v1/models catalog asked _resolves_to_resident without llama_only, so a Transformers model live from a directory that also holds GGUF exports marked a GGUF alias loaded and gave it a GGUF quant. The usage examples then pinned alias:quant that nothing could serve with switching off. Every entry in that loop is advertised as GGUF, so residency there is llama.cpp residency. The busy probe accepted any .gguf sibling while admission excludes mmproj, MTP drafters and big-endian builds. A repo holding only companions is not downloadable, so it was held at model_download_busy for the length of an unrelated download instead of falling through to the resident model as it does when no download is running. It now reuses _gguf_variants, the same filter. split_model_ref refused any slash-bearing suffix, but an unrecognized GGUF below a subdirectory keys on its path (build/llama-13b), which is_valid_gguf_variant allows and the catalog advertises. Pinning such a variant could not parse, so only the default-ranked one was reachable. A slash-bearing suffix is now a variant exactly when a real Hub repo precedes it, which still leaves C:/models/x.gguf a path rather than a quant. The usage examples treated a downloaded-but-unloaded model as runnable only under auto-switch, but a standalone UNSLOTH_MODEL_IDLE_TTL reloads exactly what it freed on the next request. The panel hid runnable examples after an idle unload. Tracked apart from auto-switch, because the stash restores the stored checkpoint only and never an arbitrary catalog entry. Also stub the index walk in the three cold-index tests that missed it: a real multi-root scan inside the cold-wait budget made them time out into a 503 under load rather than assert what they are there for. One of them flaked locally. Verified each fix is load-bearing by reverting it and watching its test fail. Backend CI command: 10195 passed, 0 failed. tsc -b and the frontend build clean. * Studio: bound the Hub admission probes and stop guessing at nested model paths Three review fixes plus a test-isolation one. _resolves_to_resident matched on a path prefix, so two separately indexed models that nest (/models/A alongside /models/A/sub/B) both satisfied it: loading B made a request for A resident and answered it with B's weights, and the catalog marked A loaded. A prefix match now counts only when no catalog entry sits deeper, which is the innermost indexed model that actually owns the file. With nothing indexed there is no nesting to tell apart, so the directory-to-weights match this exists for is unchanged. auth_check and hf_hub_download take no timeout of their own, and both ran while the provisional single-flight slot was held, so an unresponsive Hub stalled the request far past the metadata budget and reported every other model busy for the duration. Both are bounded now. Each default errs the safe way: an unchecked repo is not a cleared one, so the custom-code probe refuses on timeout, while a slow gated-repo check stays inconclusive because the download's own auth is the real gate. The usage examples caught a failed refresh into an empty catalog and a disabled auto-switch, which made a transient error authoritative and blanked every example while the model was still servable. The catalog is deliberately tri-state; a failure now keeps the last answer and retries. Also start the backend tests from a built, empty model index. Stubbing only the background warm still left the cold path walking real caches synchronously inside the admission wait, so on a large install a test asserted against a 503 "still indexing" instead of its subject. _build_index is untouched, so the tests that call it directly still exercise the real walk. Verified each fix is load-bearing by reverting it and watching its test fail. tsc -b clean. Backend CI command green apart from two failures reproduced only on this box (a real model-dir scan and an orphan-process cleanup), neither touched by this PR; staging CI is the gate for those. --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- studio/backend/auth/authentication.py | 18 +- studio/backend/core/inference/api_monitor.py | 127 +- .../backend/core/inference/llama_keepwarm.py | 18 + .../core/inference/local_model_resolver.py | 175 +- studio/backend/core/inference/model_ids.py | 25 +- .../core/inference/openai_auto_download.py | 831 ++++++++ .../hub/services/download_lifecycle.py | 27 +- .../backend/hub/services/models/downloads.py | 16 +- studio/backend/routes/inference.py | 693 ++++++- studio/backend/routes/settings.py | 14 +- studio/backend/tests/conftest.py | 22 + studio/backend/tests/test_api_monitor.py | 97 + studio/backend/tests/test_model_ids.py | 17 + .../tests/test_openai_auto_download.py | 1798 +++++++++++++++++ .../backend/tests/test_openai_auto_switch.py | 555 ++++- studio/backend/tests/test_openai_catalog.py | 163 +- .../tests/test_openai_tool_passthrough.py | 11 +- .../tests/test_training_worker_flash_attn.py | 33 + .../utils/openai_auto_switch_settings.py | 44 +- .../frontend/src/features/chat/types/api.ts | 6 + .../settings/api/openai-auto-switch.ts | 10 + .../features/settings/api/openai-models.ts | 42 + .../components/api-monitor-console.tsx | 213 +- .../components/model-auto-switch-section.tsx | 19 + .../settings/components/usage-examples.tsx | 330 ++- .../features/settings/tabs/api-keys-tab.tsx | 4 +- studio/frontend/src/i18n/locales/en.ts | 17 +- ...st_usage_examples_model_source_contract.py | 200 ++ 28 files changed, 5232 insertions(+), 293 deletions(-) create mode 100644 studio/backend/core/inference/openai_auto_download.py create mode 100644 studio/backend/tests/test_openai_auto_download.py create mode 100644 studio/frontend/src/features/settings/api/openai-models.ts create mode 100644 tests/studio/test_usage_examples_model_source_contract.py diff --git a/studio/backend/auth/authentication.py b/studio/backend/auth/authentication.py index dfb8fc513e..2481cd13e6 100644 --- a/studio/backend/auth/authentication.py +++ b/studio/backend/auth/authentication.py @@ -164,6 +164,22 @@ async def get_current_subject_allow_password_change( ) +# The literal the examples ship with; pasting one unedited is likelier than a revoked key. +API_KEY_PLACEHOLDER = f"{API_KEY_PREFIX}YOUR_KEY" + + +def _invalid_api_key_detail(token: str) -> str: + """Why the key failed. Only the unedited example placeholder is called out; + every real key still gets one indistinguishable message, so this reveals + nothing about which keys exist.""" + if token == API_KEY_PLACEHOLDER: + return ( + "This is the placeholder key from the example. Create an API key in " + f"Unsloth Studio under Settings > API and use it in place of {API_KEY_PLACEHOLDER}." + ) + return "Invalid or expired API key" + + async def _get_current_subject( credentials: HTTPAuthorizationCredentials, *, allow_password_change: bool ) -> str: @@ -176,7 +192,7 @@ async def _get_current_subject( if username is None: raise HTTPException( status_code = status.HTTP_401_UNAUTHORIZED, - detail = "Invalid or expired API key", + detail = _invalid_api_key_detail(token), ) return username diff --git a/studio/backend/core/inference/api_monitor.py b/studio/backend/core/inference/api_monitor.py index f76a38576f..2de042ab37 100644 --- a/studio/backend/core/inference/api_monitor.py +++ b/studio/backend/core/inference/api_monitor.py @@ -52,6 +52,13 @@ class ApiMonitorEntry: total_tokens: Optional[int] = None total_tokens_authoritative: bool = False error: Optional[str] = None + # "request" (HTTP call) or "lifecycle" (model load/unload: event/reason, not a prompt; shared). + kind: str = "request" + event: Optional[str] = None + reason: Optional[str] = None + shared: bool = False + # 0-100 for a running download row; None when not applicable. + progress: Optional[float] = None def snapshot(self, *, include_details: bool = True) -> dict[str, Any]: duration_ms = None @@ -85,6 +92,10 @@ class ApiMonitorEntry: "completion_tokens": self.completion_tokens, "total_tokens": self.total_tokens, "error": self.error, + "kind": self.kind, + "event": self.event, + "reason": self.reason, + "progress": self.progress, } if include_details: payload["prompt"] = self.prompt @@ -127,6 +138,75 @@ class ApiMonitor: self._trim_terminal_locked() return entry.id + def record_lifecycle( + self, + *, + event: str, + model: str, + reason: Optional[str] = None, + running: bool = False, + ) -> str: + """Record a model load/unload alongside the request traffic that caused it. + + ``running=True`` opens the row (a load in progress) and the caller closes + it with the usual :meth:`finish` / :meth:`fail`; an unload is terminal on + arrival. Rows are shared, so every subject sees them, and share the same + retention budget as requests. + """ + now = time.time() + entry = ApiMonitorEntry( + id = f"apievt_{uuid.uuid4().hex[:12]}", + endpoint = f"model.{event}", + method = "", + model = model or "default", + prompt = "", + status = "running" if running else "completed", + started_at = now, + updated_at = now, + started_monotonic = time.monotonic(), + finished_at = None if running else now, + finished_monotonic = None if running else time.monotonic(), + kind = "lifecycle", + event = event, + reason = reason, + shared = True, + ) + with self._lock: + self._entries.appendleft(entry) + self._trim_terminal_locked() + return entry.id + + def relabel(self, entry_id: Optional[str], model: str) -> None: + """Rename an open lifecycle row once the load resolves its real id (the + caller only has the load path up front, which may be an HF snapshot dir).""" + if not entry_id or not model: + return + with self._lock: + entry = self._find_locked(entry_id) + if entry is not None: + entry.model = model + entry.updated_at = time.time() + + def set_progress(self, entry_id: Optional[str], progress: Optional[float]) -> None: + """Update an open download row's percentage (clamped to 0-100).""" + if not entry_id or progress is None: + return + with self._lock: + entry = self._find_locked(entry_id) + if entry is not None and entry.status == "running": + entry.progress = min(100.0, max(0.0, float(progress))) + entry.updated_at = time.time() + + def discard(self, entry_id: Optional[str]) -> None: + """Drop a row that turned out not to be an event (a load that was already + satisfied, so nothing was actually loaded).""" + if not entry_id: + return + with self._lock: + entry = self._find_locked(entry_id) + if entry is not None: + self._entries.remove(entry) + def append_reply(self, entry_id: Optional[str], text: str) -> None: if not entry_id or not text: return @@ -212,6 +292,19 @@ class ApiMonitor: self._entries.appendleft(entry) self._trim_terminal_locked() + def fail_open(self, entry_id: Optional[str], error: str) -> None: + """Fail only a still-open row. Unlike :meth:`fail` this never touches an + entry that already finished, so a catch-all in a ``finally`` cannot stamp + an error onto a request that in fact succeeded.""" + if not entry_id: + return + with self._lock: + entry = self._find_locked(entry_id) + if entry is None or entry.finished_at is not None: + return + # Same lock as the check, so a finish() cannot land in between. + self._fail_locked(entry, error) + def fail(self, entry_id: Optional[str], error: str) -> None: if not entry_id: return @@ -224,15 +317,18 @@ class ApiMonitor: if error: entry.error = _trim(error, 1000) return - now = time.time() - entry.status = "error" - entry.error = _trim(error, 1000) - entry.updated_at = now - entry.finished_at = now - entry.finished_monotonic = time.monotonic() - self._entries.remove(entry) - self._entries.appendleft(entry) - self._trim_terminal_locked() + self._fail_locked(entry, error) + + def _fail_locked(self, entry: ApiMonitorEntry, error: str) -> None: + now = time.time() + entry.status = "error" + entry.error = _trim(error, 1000) + entry.updated_at = now + entry.finished_at = now + entry.finished_monotonic = time.monotonic() + self._entries.remove(entry) + self._entries.appendleft(entry) + self._trim_terminal_locked() def snapshot( self, @@ -244,7 +340,7 @@ class ApiMonitor: return [ entry.snapshot(include_details = include_details) for entry in self._entries - if subject is None or entry.subject == subject + if self._visible(entry, subject) ] def get( @@ -257,22 +353,29 @@ class ApiMonitor: entry = self._find_locked(entry_id) if entry is None: return None - if subject is not None and entry.subject != subject: + if not self._visible(entry, subject): return None return entry.snapshot(include_details = True) def active_count(self, *, subject: Optional[str] = None) -> int: + # Lifecycle rows show as "running" while loading but are not in-flight API requests. with self._lock: return sum( 1 for entry in self._entries - if entry.status == "running" and (subject is None or entry.subject == subject) + if entry.status == "running" + and entry.kind != "lifecycle" + and (subject is None or entry.subject == subject) ) def clear(self) -> None: with self._lock: self._entries.clear() + @staticmethod + def _visible(entry: ApiMonitorEntry, subject: Optional[str]) -> bool: + return subject is None or entry.subject == subject or entry.shared + def _find_locked(self, entry_id: str) -> Optional[ApiMonitorEntry]: for entry in self._entries: if entry.id == entry_id: diff --git a/studio/backend/core/inference/llama_keepwarm.py b/studio/backend/core/inference/llama_keepwarm.py index 3380ebf5f5..f3ec5f573f 100644 --- a/studio/backend/core/inference/llama_keepwarm.py +++ b/studio/backend/core/inference/llama_keepwarm.py @@ -345,6 +345,22 @@ def _loaded_identity(backend): return (backend.model_identifier, getattr(backend, "hf_variant", None), advertised) +def _note_idle_unload_event(freed) -> None: + """Record an idle auto-unload in the API monitor, using the advertised repo id + from the stash so the row never shows the on-disk load path. Best-effort.""" + try: + from core.inference.api_monitor import api_monitor + from core.inference.model_ids import public_model_id + + identifier, variant, advertised = (list(freed) + [None, None, None])[:3] + label = public_model_id(advertised or identifier) or "model" + if variant and ":" not in label: + label = f"{label}:{variant}" + api_monitor.record_lifecycle(event = "unload", model = label, reason = "idle") + except Exception as exc: + logger.debug("idle unload monitor event failed: %s", exc) + + async def idle_unload_loop(poll_seconds: float = 15.0) -> None: """Unload the loaded GGUF once idle past the configured TTL. Inert when off.""" from utils.openai_auto_switch_settings import ( @@ -407,6 +423,8 @@ async def idle_unload_loop(poll_seconds: float = 15.0) -> None: elif manifest: _delete_resume_files(manifest) logger.info("Idle auto-unload: freed GGUF after %ss idle", ttl) + # An idle unload stashes for reload and skips note_model_unloaded. + _note_idle_unload_event(freed) seen_model = None except Exception as exc: logger.debug("idle_unload_loop iteration failed: %s", exc) diff --git a/studio/backend/core/inference/local_model_resolver.py b/studio/backend/core/inference/local_model_resolver.py index e6014f442d..c4ac085ebe 100644 --- a/studio/backend/core/inference/local_model_resolver.py +++ b/studio/backend/core/inference/local_model_resolver.py @@ -34,6 +34,16 @@ class _LocalGgufEntry: _CACHE_TTL_S = 5.0 _lock = threading.Lock() _scan: tuple[float, dict[str, _LocalGgufEntry]] = (0.0, {}) +# Not _lock: that is held for the whole scan, so the request path would wait on it. +_warm_lock = threading.Lock() +# Repos that finished downloading but are not in the published index yet. The +# retained index covers what was already known; nothing covers the one that just +# landed until the next scan, and the request path must not call it absent. +_just_downloaded: set[str] = set() +_warming = False +_last_scan_s = 0.0 +# Rescan at most a tenth of the time: on the TTL alone a slow scan would run continuously. +_WARM_DUTY = 10.0 def _is_abs_path_id(value: str) -> bool: @@ -103,17 +113,28 @@ def _local_gguf_entry(loader_id: str, info) -> Optional[_LocalGgufEntry]: load_dir = _resolve_load_dir(p) variants, _ = list_local_gguf_variants(str(load_dir)) quants = tuple(v.quant for v in variants if getattr(v, "quant", None)) - return _LocalGgufEntry(loader_id, str(load_dir), quants) if quants else None + if not quants: + return None + # That call orders by descending size, so the head is the biggest quant, + # often F16. A bare id means whichever quant a plain load would take, so put + # that first: everything downstream reads [0], and answering with the + # largest can evict a working model and then OOM starting it. + from core.inference.openai_auto_download import preferred_quant + + best = preferred_quant(quants) + if best and quants[0] != best: + quants = (best, *(q for q in quants if q != best)) + return _LocalGgufEntry(loader_id, str(load_dir), quants) except Exception: return None -def info_has_local_gguf(info) -> bool: - """True when *info* (a LocalModelInfo) points to on-disk GGUF weights the - auto-switch path can load. Read from the files, not ``info.model_format``: the - HF-cache scanner leaves model_format unset for GGUF snapshots, so a - model_format filter would drop every cached GGUF. Lets /v1/models advertise - exactly what /v1 can serve.""" +def local_gguf_quants(info) -> Optional[tuple[str, ...]]: + """On-disk quant labels for *info*, or None when it is not a servable local + GGUF. Read from the files, not ``info.model_format``: the HF-cache scanner + leaves model_format unset for GGUF snapshots, so a model_format filter would + drop every cached GGUF. Lets /v1/models advertise exactly what /v1 can serve, + and which quant to name, from a single scan.""" from pathlib import Path path = getattr(info, "path", None) @@ -123,8 +144,14 @@ def info_has_local_gguf(info) -> bool: if isinstance(path, str) and any( seg in (".studio_links", "ollama_links") for seg in Path(path).parts ): - return False - return _local_gguf_entry(getattr(info, "id", "") or "", info) is not None + return None + entry = _local_gguf_entry(getattr(info, "id", "") or "", info) + return entry.variants if entry is not None else None + + +def info_has_local_gguf(info) -> bool: + """True when *info* points to on-disk GGUF weights the auto-switch path can load.""" + return local_gguf_quants(info) is not None def _build_index() -> dict[str, _LocalGgufEntry]: @@ -287,6 +314,36 @@ def _sibling_revision_entries(raw_id: str, loader_id: str): yield sibling.name, entry +def note_downloaded(repo_id: Optional[str]) -> None: + """Record a repo as present ahead of the scan that will index it.""" + if not repo_id: + return + with _lock: + _just_downloaded.add(repo_id.strip().lower()) + + +def recently_downloaded(repo_id: str) -> bool: + """Whether *repo_id* finished downloading since the last completed scan.""" + if not isinstance(repo_id, str) or not repo_id.strip(): + return False + return repo_id.strip().lower() in _just_downloaded + + +def invalidate_index() -> None: + """Mark the cached scan stale so the next resolve sees a just-finished + download, rather than waiting out the TTL. + + Keeps the entries. Callers on the request path read this cache without + scanning, so emptying it would leave them with no evidence about any local + model until the rebuild lands, and a bare request for one of them would be + answered by whatever is resident. Only a completed download invalidates, and + that only ever adds models, so the retained entries stay true. + """ + global _scan + with _lock: + _scan = (0.0, _scan[1]) + + def _index() -> dict[str, _LocalGgufEntry]: global _scan # Build under the lock so concurrent callers with an expired cache don't all @@ -301,23 +358,78 @@ def _index() -> dict[str, _LocalGgufEntry]: # an install with many local models can itself exceed the TTL, which would # store the cache already expired and make every request rebuild the index. _scan = (time.monotonic(), fresh) + # The scan supersedes the notes: whatever landed is in the index now. + _just_downloaded.clear() return fresh -def resolve_local_gguf(requested: str) -> Optional[tuple[str, Optional[str], str]]: +def index_is_built() -> bool: + """Whether a scan has ever completed, freshness aside. + + Lock-free on purpose: ``_lock`` is held for the whole scan, so taking it here + would park the request path on the very scan it is trying to stay off. Reading + ``_scan[0]`` is safe because ``_scan`` is only ever rebound, never mutated. + """ + return bool(_scan[0]) + + +def warm_index_soon() -> None: + """(Re)build the index off the request path when it is missing or past its TTL. + + Callers that cannot afford the scan use this plus ``allow_scan=False``, so this + is the only thing that ever refreshes the index for them. It has to cover a + stale index and not just an absent one: a model downloaded through the Hub UI + or dropped into a scan folder has no invalidation hook, and would otherwise stay + invisible to those callers for the life of the process. + + Never touches ``_lock``, which the scan holds throughout, and never blocks. + """ + global _warming + if time.monotonic() - _scan[0] < max(_CACHE_TTL_S, _last_scan_s * _WARM_DUTY): + return + with _warm_lock: + if _warming: + return + _warming = True + + def _run() -> None: + global _warming, _last_scan_s + started = time.monotonic() + try: + _index() + except Exception: + pass + finally: + _last_scan_s = time.monotonic() - started + with _warm_lock: + _warming = False + + threading.Thread(target = _run, name = "local-model-index-warm", daemon = True).start() + + +def resolve_local_gguf( + requested: str, *, allow_scan: bool = True +) -> Optional[tuple[str, Optional[str], str]]: """Return ``(load_path, gguf_variant, loader_id)`` for a local match, else None. ``load_path`` is the concrete on-disk path to hand /load (so it never fetches a remote), ``loader_id`` is the advertised id used as the launch-override key. ``requested`` is ``repo`` or ``repo:VARIANT``. An exact id match wins first (so ids containing a colon still resolve); else the last ``:VARIANT`` is split - off and resolves only when that quant is on disk. + off and resolves only when that quant is on disk, unless it names no quant at + all (an Ollama-style ":latest"), which means the repo. + + ``allow_scan=False`` answers from the last built index and never rebuilds, + for callers on the request path: the scan walks several model dirs and HF + caches, takes seconds on a large install, and holds a lock every other + caller queues behind. A stale answer is fine there, since what is on disk + barely moves and a finished download calls :func:`invalidate_index`. """ if not isinstance(requested, str) or not requested.strip(): return None requested = requested.strip() try: - index = _index() + index = _index() if allow_scan else _scan[1] entry = index.get(requested.lower()) if entry is not None: variant = entry.variants[0] if entry.variants else None @@ -333,8 +445,45 @@ def resolve_local_gguf(requested: str) -> Optional[tuple[str, Optional[str], str for v in entry.variants: if v.lower() == wanted: return entry.load_path, v, entry.loader_id - return None + from core.inference.openai_auto_download import looks_like_quant + + if looks_like_quant(variant): + return None + # ":latest" or ":8b" names no file, so it means the repo; a real quant that + # is not on disk still misses, or a swap would serve the wrong weights. + return entry.load_path, (entry.variants[0] if entry.variants else None), entry.loader_id except Exception: # Best-effort: any resolver failure falls through to the loaded model, # so a malformed name can never turn a servable request into a 500. return None + + +MISS_MODEL_NOT_FOUND = "model_not_found" +MISS_VARIANT_NOT_FOUND = "variant_not_found" + + +def describe_local_miss(requested: str) -> tuple[str, tuple[str, ...]]: + """Why :func:`resolve_local_gguf` missed, so an error can say "wrong quant" + instead of "no such model". + + ``(MISS_VARIANT_NOT_FOUND, )`` when the repo is downloaded but + the requested ``:VARIANT`` is not, else ``(MISS_MODEL_NOT_FOUND, ())``. Splits + the name like the resolver so the two agree. Fail-safe: a scan failure reports + the generic miss rather than raising into the handler. + """ + if not isinstance(requested, str) or not requested.strip(): + return MISS_MODEL_NOT_FOUND, () + base, sep, variant = requested.strip().rpartition(":") + from core.inference.openai_auto_download import looks_like_quant + + # Split like the resolver or the two disagree: a tag naming no quant means the + # repo there, so reporting a missing quant for it would name one nobody asked for. + if not sep or not looks_like_quant(variant): + return MISS_MODEL_NOT_FOUND, () + try: + entry = _index().get(base.strip().lower()) + except Exception: + return MISS_MODEL_NOT_FOUND, () + if entry is None or not entry.variants: + return MISS_MODEL_NOT_FOUND, () + return MISS_VARIANT_NOT_FOUND, entry.variants diff --git a/studio/backend/core/inference/model_ids.py b/studio/backend/core/inference/model_ids.py index 548cc60f94..a6270b955e 100644 --- a/studio/backend/core/inference/model_ids.py +++ b/studio/backend/core/inference/model_ids.py @@ -39,10 +39,30 @@ def _looks_like_path(identifier: str) -> bool: return False +def hf_cache_repo_id(path: Optional[str]) -> Optional[str]: + """``.../models--org--name/snapshots/`` -> ``org/name``, else None. + + A model loaded straight out of the HF cache has a snapshot directory as its + identifier, whose basename is a commit hash. Recover the repo id so callers + show ``unsloth/gemma-4-31B-it-GGUF`` rather than ``c1ac76e99d55...``. + """ + if not path: + return None + parts = str(path).replace("\\", "/").split("/") + for index, part in enumerate(parts): + # Only inside the real cache layout: a "models--" name alone is not a repo id. + if part.startswith("models--") and parts[index + 1 : index + 2] == ["snapshots"]: + return part[len("models--") :].replace("--", "/") + return None + + def public_model_id(identifier: Optional[str]) -> Optional[str]: """Return a clean, path-free public id for *identifier*. - - Local GGUF path -> the file stem with ``.gguf`` stripped, e.g. + - HF cache path -> the repo id it came from, e.g. + ``~/.cache/huggingface/hub/models--unsloth--X-GGUF/snapshots/`` -> + ``unsloth/X-GGUF``. + - Other local GGUF path -> the file stem with ``.gguf`` stripped, e.g. ``/srv/models/Qwen3-30B-A3B-Q4_K_M.gguf`` -> ``Qwen3-30B-A3B-Q4_K_M``. - HF repo id (``org/model``) and already-clean names -> returned unchanged. - ``None`` / empty -> returned unchanged. @@ -51,6 +71,9 @@ def public_model_id(identifier: Optional[str]) -> Optional[str]: return identifier if not _looks_like_path(identifier): return identifier + repo_id = hf_cache_repo_id(identifier) + if repo_id: + return repo_id name = os.path.basename(identifier.replace("\\", "/").rstrip("/")) if name.lower().endswith(_GGUF_SUFFIX): name = name[: -len(_GGUF_SUFFIX)] diff --git a/studio/backend/core/inference/openai_auto_download.py b/studio/backend/core/inference/openai_auto_download.py new file mode 100644 index 0000000000..fee87a42f2 --- /dev/null +++ b/studio/backend/core/inference/openai_auto_download.py @@ -0,0 +1,831 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Opt-in: fetch a GGUF a /v1 request names but this server doesn't have. + +Auto-switch only loads models already on disk. With +``openai_api_auto_download_model`` on, a miss that looks like a real Hub repo is +downloaded in the background instead of erroring, and the request is told to +retry rather than being held open: a quant is routinely tens of GB, far longer +than any client (or the Cloudflare edge on ``--secure``) will wait, and the +inference lifecycle gate must not be held meanwhile. The resident model keeps +serving throughout, and the retry that lands after the download is served by the +new model through the ordinary auto-switch path. + +Admission is deliberately narrow, since a request only needs an API key: +- ``namespace/name`` only, and only when the Hub confirms it is a GGUF repo. + ``gpt-4`` and ``anthropic/claude-3.5-sonnet`` alike fall through to the + resident model as before: a namespace is not evidence of intent, since LiteLLM + and OpenRouter address every provider that way. +- GGUF repos only, decided from the remote file list, not the repo name. GGUF + runs under llama.cpp, which never imports repo Python. +- Anything declaring ``auto_map`` is refused, so ``trust_remote_code`` can only + ever be granted deliberately in the UI, never by an API call. +- One download at a time, so a key holder cannot fan out fetches. +""" + +from __future__ import annotations + +import asyncio +import shutil +import threading +import time +from dataclasses import dataclass +from typing import Optional + +from loggers import get_logger + +logger = get_logger(__name__) + +# Keep the Hub probe short so a slow Hub can't stall the request path. +_MODEL_INFO_TIMEOUT_S = 8.0 +# auth_check and hf_hub_download take no timeout of their own, and both run while the +# provisional slot is held, so an unresponsive Hub would pin the single flight and stall +# the request long past the metadata budget. The code probe fetches up to three small +# configs, so it gets more room than the single auth call. +_CODE_PROBE_TIMEOUT_S = 20.0 +# Headroom left free after the download, so filling the disk can't wedge the box. +_DISK_RESERVE_BYTES = 5 * 1024**3 +_WATCH_POLL_S = 2.0 +# A stalled watcher must not pin the single-flight slot forever. +_MAX_WATCH_S = 24 * 60 * 60 +# Past the watch window the row is already resolved, so poll only to see whether +# the worker is still alive and still owns the slot. +_TIMED_OUT_POLL_S = 60.0 +_RETRY_AFTER_S = 30 +# Long enough for a client honouring Retry-After to come back and be told, short +# enough that a client that never returns cannot hold the slot. +_FAILED_HOLD_S = 3 * _RETRY_AFTER_S +_MAX_LISTED_VARIANTS = 8 + + +@dataclass(frozen = True) +class AutoDownloadRefusal: + """Why this request cannot be served yet. The route turns it into an + HTTPException with the surface's own error envelope.""" + + status: int + code: str + message: str + retry_after: Optional[int] = None + + +@dataclass +class _Active: + repo_id: str + # None while the Hub probe is still deciding which quant to fetch. + variant: Optional[str] = None + expected_bytes: int = 0 + monitor_id: Optional[str] = None + started_at: float = 0.0 + # Set when the worker failed. The slot is kept until a retry surfaces it, since + # the advertised retry interval is far longer than the watcher's poll and the + # client would otherwise just restart the same failing download. + error: Optional[str] = None + failed_at: float = 0.0 + + +_lock = threading.Lock() +_active: Optional[_Active] = None + +# Repos the Hub says are not servable, so a "vendor/model" miss doesn't re-probe every request. +_NOT_SERVABLE_TTL_S = 10 * 60 +_NOT_SERVABLE_MAX = 256 +_cache_lock = threading.Lock() +_not_servable: dict[str, float] = {} + + +def _public_label(repo_id: str, variant: Optional[str]) -> str: + return f"{repo_id}:{variant}" if variant else repo_id + + +def split_model_ref(requested: str) -> tuple[str, Optional[str]]: + """``org/repo:QUANT`` -> ``("org/repo", "QUANT")``; no suffix -> variant None. + + Splits on the last colon. A slash-bearing suffix is only a variant when a real + Hub repo precedes it: an unrecognized GGUF below a subdirectory keys on its path + ("build/llama-13b", which is_valid_gguf_variant allows and the catalog advertises), + while "C:/models/x.gguf" leaves a drive letter that is no repo id at all. + """ + text = (requested or "").strip() + base, sep, suffix = text.rpartition(":") + if not sep or not base or not suffix: + return text, None + stripped = base.strip() + if "/" in suffix: + from hub.utils.paths import is_valid_repo_id + if "/" not in stripped or not is_valid_repo_id(stripped): + return text, None + return stripped, suffix.strip() + + +def is_downloadable_ref(requested: str) -> bool: + """Whether *requested* is shaped like a Hub repo we may fetch. + + Requires an explicit namespace. That keeps ``gpt-4`` and other foreign ids + falling through untouched, and avoids the bare-name ``unsloth/`` prefixing in + ModelConfig.from_identifier turning an unrelated label into a real repo. + """ + from hub.utils.paths import is_valid_repo_id + + repo_id, variant = split_model_ref(requested) + if "/" not in repo_id or not is_valid_repo_id(repo_id): + return False + if variant is not None: + from hub.utils.paths import is_valid_gguf_variant + return is_valid_gguf_variant(variant) + return True + + +def looks_like_quant(variant: Optional[str]) -> bool: + """Whether a ``:suffix`` names a GGUF quant rather than a foreign tag. + + ``vendor/model`` is how LiteLLM and OpenRouter address every provider, and + ``name:latest`` is how Ollama tags one, so neither a namespace nor a colon + proves a request was meant for this server. A real quant label does. + """ + import re + + from utils.models.model_config import _GGUF_KNOWN_QUANT_RE + + if not variant: + return False + # _extract_quant_label can append a bpw modifier (IQ4_XS-3.53bpw); still a quant. + label = re.sub(r"-[0-9]+(?:\.[0-9]+)?bpw$", "", variant.strip(), flags = re.IGNORECASE) + return _GGUF_KNOWN_QUANT_RE.fullmatch(label) is not None + + +def _hub_token(hf_token: Optional[str]): + """The caller's token, or an explicit False. + + None makes huggingface_hub fall back to a cached login, which here would be + the server owner's. False is what actually means anonymous. + """ + return hf_token or False + + +def _servable_key(repo_id: str, hf_token: Optional[str]) -> str: + """Cache key, per credential. + + The Hub answers 404 for a private repo the caller cannot see, so a verdict + reached without a token says nothing about a caller who has one. Keyed on a + digest so the token itself is never held here. + """ + import hashlib + + seen_as = hashlib.sha256(hf_token.encode()).hexdigest()[:16] if hf_token else "anon" + return f"{repo_id.lower()}\n{seen_as}" + + +def _mark_not_servable(repo_id: str, hf_token: Optional[str]) -> None: + with _cache_lock: + if len(_not_servable) >= _NOT_SERVABLE_MAX: + _not_servable.clear() + _not_servable[_servable_key(repo_id, hf_token)] = time.monotonic() + _NOT_SERVABLE_TTL_S + + +def _is_not_servable(repo_id: str, hf_token: Optional[str]) -> bool: + key = _servable_key(repo_id, hf_token) + with _cache_lock: + expires = _not_servable.get(key) + if expires is None: + return False + if expires <= time.monotonic(): + del _not_servable[key] + return False + return True + + +def _gated_refusal(repo_id: str) -> AutoDownloadRefusal: + return AutoDownloadRefusal( + status = 403, + code = "model_access_denied", + message = ( + f"'{repo_id}' is gated on Hugging Face. Accept its licence, then retry with " + "your own token in the X-Unsloth-HF-Token header: automatic download never " + "uses this server's Hugging Face identity." + ), + ) + + +async def _bounded_probe(fn, *args, timeout: float, default): + """Run a blocking Hub probe off the loop, bounding only the wait. + + The thread is left to finish (a blocking socket read cannot be cancelled); the + caller stops waiting and takes *default*, which each call site chooses so that a + timeout errs the safe way. + """ + try: + return await asyncio.wait_for(asyncio.to_thread(fn, *args), timeout) + except (TimeoutError, asyncio.TimeoutError): + logger.debug("hub probe %s timed out after %ss", getattr(fn, "__name__", fn), timeout) + return default + + +def _auth_denied(repo_id: str, hf_token: Optional[str]) -> bool: + """Whether this token lacks file access to a gated repo. False when the + check is inconclusive: the download's own auth is the real gate.""" + from hub.utils.hf_errors import hf_error_status + + try: + from huggingface_hub import auth_check + auth_check(repo_id, token = _hub_token(hf_token)) + except Exception as exc: + return hf_error_status(exc) in (401, 403) + return False + + +def _gguf_variants(siblings) -> dict[str, int]: + """Quant label -> bytes the download will actually fetch. + + Mirrors list_gguf_variants for the selectable labels: companions (mmproj/MTP) + and big-endian builds are not quants of their own, and sharded quants sum + across their shards. The byte total comes from the download plan, which folds + the companions back into every quant, so the disk reserve is measured against + what the worker fetches rather than the main files alone. + """ + from hub.utils.gguf import extract_quant_label as canonical_quant_label + from hub.utils.gguf_plan import build_gguf_variant_plans + from utils.models.model_config import ( + _extract_quant_label, + _is_big_endian_gguf_path, + _is_mmproj, + _is_mtp_drafter, + ) + + siblings = list(siblings or []) + plans = build_gguf_variant_plans(siblings) + sizes: dict[str, int] = {} + for sibling in siblings: + name = getattr(sibling, "rfilename", "") or "" + if not name.lower().endswith(".gguf"): + continue + quant = _extract_quant_label(name) + if not looks_like_quant(quant): + # With no recognized quant token the two extractors part ways: this one + # takes the last hyphenated segment ("7b" of llama-7b) while the plan and + # the worker key the whole stem. Advertising ours dispatches a variant the + # worker cannot resolve, so take theirs for the unrecognized case only. + quant = canonical_quant_label(name) or quant + if _is_mmproj(name) or _is_mtp_drafter(name) or _is_big_endian_gguf_path(name, quant): + continue + plan = plans.get(quant.lower()) + if plan is not None: + sizes[quant] = plan.download_size_bytes + else: + sizes[quant] = sizes.get(quant, 0) + int(getattr(sibling, "size", 0) or 0) + return sizes + + +def _remaining_bytes(repo_id: str, plan, expected_bytes: int) -> int: + """Bytes still to fetch: a resumed quant or a companion shared with another + quant is already on disk, and charging for it can 507 a download that fits.""" + try: + from hub.utils.download_registry import existing_blob_bytes + + hashes = frozenset( + file.sha256 for file in getattr(plan, "expected_files", ()) or () if file.sha256 + ) + if not hashes: + return expected_bytes + return max(0, expected_bytes - existing_blob_bytes("model", repo_id, hashes)) + except Exception: + return expected_bytes + + +def _enough_disk(need_bytes: int) -> tuple[bool, int]: + """(fits, free_bytes). Fail-open on an unreadable cache root: the download + worker runs its own preflight, this only adds the reserve margin.""" + try: + from hub.utils.hf_cache_state import hf_cache_root + + root = hf_cache_root(create = True) + if root is None: + return True, 0 + free = shutil.disk_usage(root).free + except Exception: + return True, 0 + return free >= need_bytes + _DISK_RESERVE_BYTES, free + + +def _gb(num_bytes: int) -> str: + return f"{num_bytes / 1024**3:.1f} GB" + + +async def _job_state(repo_id: str, variant: Optional[str]) -> tuple[str, Optional[str]]: + from hub.services.models import downloads + try: + status = await downloads.get_download_status_response(repo_id, variant or "") + return status.state, status.error + except Exception as exc: + # "unknown", not "idle": idle ends the watch, and a failed probe proves nothing. + logger.debug("auto-download: status probe failed for %r: %s", repo_id, exc) + return "unknown", None + + +async def _progress_percent( + repo_id: str, variant: Optional[str], expected_bytes: int, hf_token: Optional[str] +) -> Optional[float]: + """0-100, or None. The hub service reports a 0-1 fraction, so scale it.""" + from hub.services.models import downloads + try: + payload = await downloads.get_gguf_download_progress_response( + repo_id, variant or "", expected_bytes, hf_token + ) + fraction = payload.get("progress") + if not isinstance(fraction, (int, float)): + return None + return min(100.0, max(0.0, float(fraction) * 100.0)) + except Exception: + return None + + +def _release(active: Optional[_Active]) -> None: + """Free the single-flight slot, but only while *active* still owns it. + + Keying the release on ``repo_id`` alone let a stale operation clear a newer + one for the same repo: variant A errors, an adopting request frees the slot, + a retry starts variant B, and A's watcher then matches on the repo and clears + B on its way out -- admitting a second repository download alongside B. + Identity ties every release to the operation that actually took the slot. + """ + global _active + if active is None: + return + with _lock: + if _active is active: + _active = None + + +async def _watch(active: _Active, hf_token: Optional[str]) -> None: + """Poll a dispatched job so the monitor row resolves and the resolver cache + is dropped the moment the weights land.""" + from core.inference import api_monitor as monitor_module + + api_monitor = monitor_module.api_monitor + deadline = time.monotonic() + _MAX_WATCH_S + timed_out = False + try: + while True: + await asyncio.sleep(_TIMED_OUT_POLL_S if timed_out else _WATCH_POLL_S) + state, error = await _job_state(active.repo_id, active.variant) + if state in ("running", "cancelling", "unknown"): + if timed_out: + # A worker still running still owns the slot: releasing it on the + # clock alone would admit a second multi-GB download alongside it. + # "unknown" cannot confirm it is alive, so stop holding it then, + # or a broken probe would wedge auto-download for good. + if state == "unknown": + return + continue + if time.monotonic() >= deadline: + api_monitor.fail_open(active.monitor_id, "Download timed out") + timed_out = True + continue + # Only "running" has progress; the others are still in flight, so keep the slot. + if state == "running": + api_monitor.set_progress( + active.monitor_id, + await _progress_percent( + active.repo_id, active.variant, active.expected_bytes, hf_token + ), + ) + continue + if state == "cancelled": + api_monitor.finish(active.monitor_id, status = "cancelled") + return + if state == "complete": + # No invalidate here: finalize_worker_exit already dropped the cache and + # started the warm, and a second one would mark that fresh scan stale and + # push a synchronous rescan onto the client's retry. + api_monitor.finish(active.monitor_id, status = "completed") + elif state == "idle": + # The job vanished without a terminal state (worker killed). + api_monitor.fail_open(active.monitor_id, "Download did not complete") + else: + api_monitor.fail_open(active.monitor_id, error or f"Download {state}") + # Keep the slot so the next retry is told it failed rather than + # silently starting the same download again. + active.error = error or f"Download {state}" + active.failed_at = time.monotonic() + return + return + except asyncio.CancelledError: + raise + except Exception as exc: + logger.warning("auto-download: watcher failed for %r: %s", active.repo_id, exc) + api_monitor.fail_open(active.monitor_id, "Download tracking failed") + finally: + if not active.failed_at: + _release(active) + + +def _downloading_refusal(label: str, percent: Optional[float]) -> AutoDownloadRefusal: + progress = f" ({percent:.0f}% done)" if percent is not None else "" + return AutoDownloadRefusal( + status = 503, + code = "model_downloading", + message = (f"Downloading '{label}'{progress}. Retry shortly. Track it in Unsloth Studio."), + retry_after = _RETRY_AFTER_S, + ) + + +async def _is_downloadable_model(repo_id: str, hf_token: Optional[str]) -> bool: + """Whether the Hub has this repo with GGUF weights we could fetch. + + Only asked while another download holds the slot, to tell a second download + apart from an ordinary foreign label. Any failure answers False: falling + through to the resident model is what such a label does anyway, and refusing + it would strand normal traffic for the length of the download. + """ + if _is_not_servable(repo_id, hf_token): + return False + + def _probe(): + from huggingface_hub import HfApi + return HfApi(token = _hub_token(hf_token)).model_info(repo_id, timeout = _MODEL_INFO_TIMEOUT_S) + + try: + info = await asyncio.to_thread(_probe) + except Exception: + return False + # The same filter admission uses, not a bare .gguf test: mmproj, MTP drafters and + # big-endian builds are companions rather than quants, so a repo holding only those + # is not downloadable here either. Answering otherwise would hold an ordinary + # foreign label at model_download_busy for the length of an unrelated download. + servable = bool(_gguf_variants(getattr(info, "siblings", None))) + if not servable: + _mark_not_servable(repo_id, hf_token) + return servable + + +async def maybe_auto_download( + requested_model: str, + *, + hf_token: Optional[str] = None, + require_vision: bool = False, +) -> Optional[AutoDownloadRefusal]: + """Start (or report on) a background fetch of *requested_model*. + + Returns None when the request should carry on unchanged, or a refusal the + caller must raise. Only called after the local resolver has already missed. + + ``require_vision`` refuses a target with no mmproj companion rather than + spending gigabytes on weights that cannot answer the request that asked for + them; the local capability guard only ever sees an already-downloaded model. + """ + global _active + + repo_id, wanted_variant = split_model_ref(requested_model) + if not is_downloadable_ref(requested_model): + return None + if _is_not_servable(repo_id, hf_token) and not looks_like_quant(wanted_variant): + return None + + # Settle the single-flight slot before the network, so retries during a download stay cheap. + busy: Optional[_Active] = None + with _lock: + current = _active + if current is not None and current.failed_at: + # A held failure only owns the slot until someone is told about it. + if current.repo_id != repo_id and time.monotonic() - current.failed_at > _FAILED_HOLD_S: + _active = current = None + if current is not None and current.repo_id == repo_id: + adopted = current + elif current is not None: + adopted = None + busy = current + else: + adopted = None + provisional = _Active(repo_id = repo_id, started_at = time.time()) + _active = provisional + + if busy is not None: + # Refusing before the probe blocks ordinary drop-in traffic: a namespaced label + # that is not a downloadable GGUF repo (LiteLLM/OpenRouter style) would be told + # to wait out a multi-hour download instead of falling through to the resident + # model. Only a label that could itself be downloaded is a second download. + if not await _is_downloadable_model(repo_id, hf_token): + return None + return AutoDownloadRefusal( + status = 503, + code = "model_download_busy", + message = ( + f"Already downloading '{_public_label(busy.repo_id, busy.variant)}'. " + f"Retry '{requested_model}' once it finishes." + ), + retry_after = _RETRY_AFTER_S, + ) + + if adopted is not None: + if adopted.variant is None: + # Still probing: no job yet, and a stale whole-repo error would free the probe's slot. + return _downloading_refusal(adopted.repo_id, None) + state, error = await _job_state(adopted.repo_id, adopted.variant) + if state in ("running", "cancelling", "unknown"): + return _downloading_refusal( + _public_label(adopted.repo_id, adopted.variant), + await _progress_percent( + adopted.repo_id, adopted.variant, adopted.expected_bytes, hf_token + ), + ) + if state == "error" or adopted.error: + error = error or adopted.error + # Surface once, then free the slot so a retry can start over. + _release(adopted) + return AutoDownloadRefusal( + status = 502, + code = "model_download_failed", + message = f"Downloading '{requested_model}' failed: {error or 'unknown error'}", + ) + # complete/idle/cancelled: the watcher is about to free the slot, so retry once more. + return _downloading_refusal( + _public_label(adopted.repo_id, adopted.variant), + 100.0 if state == "complete" else None, + ) + + try: + return await _admit_and_start( + repo_id, wanted_variant, requested_model, hf_token, provisional, require_vision + ) + except BaseException: + # Not `except Exception`: a cancel mid-probe would otherwise wedge the provisional slot. + _release(provisional) + raise + + +async def _admit_and_start( + repo_id: str, + wanted_variant: Optional[str], + requested_model: str, + hf_token: Optional[str], + active: _Active, + require_vision: bool = False, +) -> Optional[AutoDownloadRefusal]: + from hub.utils.hf_errors import hf_error_status + + def _probe(): + from huggingface_hub import HfApi + return HfApi(token = _hub_token(hf_token)).model_info( + repo_id, files_metadata = True, timeout = _MODEL_INFO_TIMEOUT_S + ) + + try: + info = await asyncio.to_thread(_probe) + except Exception as exc: + _release(active) + status = hf_error_status(exc) + if status == 401: + return AutoDownloadRefusal( + status = 401, + code = "model_access_denied", + message = ( + f"Hugging Face rejected the token sent for '{repo_id}'. Replace the " + "X-Unsloth-HF-Token header with a valid token; retrying will not help." + ), + ) + if status == 403: + return _gated_refusal(repo_id) + if status == 404: + _mark_not_servable(repo_id, hf_token) + # Unknown to the Hub reads as a foreign label; only an explicit quant makes it ours. + if not looks_like_quant(wanted_variant): + return None + # A private repo reads as absent without a token; don't confirm either way. + return AutoDownloadRefusal( + status = 404, + code = "model_not_found", + message = ( + f"'{repo_id}' was not found on Hugging Face, or is not accessible. " + "If it is private, send a token in the X-Unsloth-HF-Token header." + ), + ) + logger.warning("auto-download: Hub lookup failed for %r: %s", repo_id, exc) + return AutoDownloadRefusal( + status = 503, + code = "model_lookup_failed", + message = f"Could not reach Hugging Face to look up '{repo_id}'. Retry shortly.", + retry_after = _RETRY_AFTER_S, + ) + + # Inconclusive on timeout: the download's own auth is the real gate. + if getattr(info, "gated", False) and await _bounded_probe( + _auth_denied, repo_id, hf_token, timeout = _MODEL_INFO_TIMEOUT_S, default = False + ): + # Metadata for a gated repo is not file access; unchecked, the config read below lies. + _release(active) + return _gated_refusal(repo_id) + + variants = _gguf_variants(getattr(info, "siblings", None)) + if not variants: + _release(active) + _mark_not_servable(repo_id, hf_token) + if not looks_like_quant(wanted_variant): + return None + return AutoDownloadRefusal( + status = 400, + code = "model_not_supported", + message = ( + f"'{repo_id}' has no GGUF weights. Automatic download serves GGUF only; " + "load other formats from Unsloth Studio." + ), + ) + + # trust_remote_code gate: _config_has_auto_map is tri-state, so refuse on True and on None. + from utils.security.consent import _config_has_auto_map + + # _hub_token, not the raw token: None lets huggingface_hub fall back to a cached + # server login, so a caller-named repo would be probed with this server's identity. + # Same rule as the metadata probe and the worker. + # None on timeout, which refuses: an unchecked repo is not a cleared one. + has_auto_map = await _bounded_probe( + _config_has_auto_map, + repo_id, + _hub_token(hf_token), + timeout = _CODE_PROBE_TIMEOUT_S, + default = None, + ) + if has_auto_map is not False: + _release(active) + unknown = has_auto_map is None + return AutoDownloadRefusal( + status = 403, + code = "remote_code_consent_required", + message = ( + f"'{repo_id}' " + + ( + "could not be checked for custom code" + if unknown + else "ships custom code that runs on load" + ) + + ". Load it once in Unsloth Studio to review and approve it, then retry." + ), + ) + + variant = _match_variant(wanted_variant, variants) + if variant is None: + _release(active) + listed = sorted(variants) + shown = ", ".join(listed[:_MAX_LISTED_VARIANTS]) + extra = len(listed) - _MAX_LISTED_VARIANTS + return AutoDownloadRefusal( + status = 404, + code = "model_not_found", + message = ( + f"'{repo_id}' has no quant '{wanted_variant}'. Available quants: " + f"{shown}{f' and {extra} more' if extra > 0 else ''}." + ), + ) + + expected_bytes = variants[variant] + from hub.utils.gguf_plan import build_gguf_variant_plans + + plan = build_gguf_variant_plans(list(getattr(info, "siblings", None) or [])).get( + variant.lower() + ) + if require_vision and not (plan and plan.mmproj_filenames): + _release(active) + return AutoDownloadRefusal( + status = 400, + code = "invalid_value", + message = ( + f"'{_public_label(repo_id, variant)}' ships no mmproj companion, so it " + "cannot answer the image or audio input in this request. It was not " + "downloaded." + ), + ) + + need_bytes = _remaining_bytes(repo_id, plan, expected_bytes) + fits, free = _enough_disk(need_bytes) + if not fits: + _release(active) + return AutoDownloadRefusal( + status = 507, + code = "insufficient_disk_space", + message = ( + f"'{_public_label(repo_id, variant)}' needs {_gb(need_bytes)} plus " + f"{_gb(_DISK_RESERVE_BYTES)} headroom, but only {_gb(free)} is free." + ), + ) + + return await _dispatch(repo_id, variant, expected_bytes, requested_model, hf_token, active) + + +def preferred_quant(labels) -> Optional[str]: + """The quant a plain load would pick from *labels*, or None. + + The one ranking for "which quant did they mean": local resolution, remote + admission and what /v1/models advertises all have to agree, or a bare id + means a different quant depending on which of them answered it. + """ + from utils.models.model_config import _pick_best_gguf + + # _pick_best_gguf ranks filenames and matches upper-case tokens, so feed "