Merge pull request #136 from unslothai/setup/update-setup-sh-dependencies

setup.sh: Replace inline pip installs with pinned requirements files

.gitignore

@@ -18,6 +18,7 @@ unsloth_compiled_cache/
 
 # ML artifacts (large files)
 outputs/
+exports/
 *.gguf
 *.safetensors
 

requirements/base.txt

@@ -0,0 +1,3 @@
+# Core unsloth packages
+unsloth-zoo
+unsloth

requirements/extras-no-deps.txt

@@ -0,0 +1,13 @@
+# Audio extras (installed with --no-deps --no-cache-dir)
+descript-audio-codec
+descript-audiotools
+julius
+torchcodec
+snac
+
+# TRL and related packages
+trl==0.23.1
+git+https://github.com/meta-pytorch/OpenEnv.git
+executorch==1.0.1
+torch-c-dlpack-ext
+sentence_transformers==5.2.0

requirements/extras.txt

@@ -0,0 +1,56 @@
+# OpenEnv dependencies
+tomli
+tomli-w
+
+# ExecuTorch dependencies
+ruamel.yaml
+coremltools
+expecttest
+flatbuffers
+hydra-core
+hypothesis
+kgb
+parameterized
+pytest<9.0
+pytest-json-report
+pytest-rerunfailures==15.1
+pytest-xdist
+# Also needed by sentence_transformers
+scikit-learn==1.7.1
+
+# Additional extras
+pybind11
+langid
+jiwer
+omegaconf
+einx
+pyloudnorm
+openai-whisper
+uroman
+MeCab
+loguru
+flatten_dict
+ffmpy
+randomname
+argbind
+tiktoken
+ftfy
+importlib-resources
+librosa
+markdown2
+matplotlib
+pystoi
+soundfile
+tensorboard
+torch-stoi
+evaluate
+timm
+transformers-cfg
+open_spiel
+addict
+easydict
+einops
+tabulate
+fastmcp>=2.0.0
+openai>=2.7.2
+websockets>=13.0,<14

requirements/overrides.txt

@@ -0,0 +1,7 @@
+# Torch AO overrides (installed with --force-reinstall --no-cache-dir)
+torchao==0.14.0
+transformers==4.57.1
+pytorch_tokenizers
+
+# Kernel packages
+kernels

requirements/studio.txt

@@ -0,0 +1,13 @@
+# Studio UI backend dependencies
+typer
+fastapi
+uvicorn
+pydantic
+matplotlib
+pandas
+nest_asyncio
+datasets==4.3.0
+pyjwt
+easydict
+addict
+gradio>=4.0.0

requirements/triton-kernels.txt

@@ -0,0 +1,2 @@
+# Triton kernels (installed with --no-deps, from source)
+triton_kernels @ git+https://github.com/triton-lang/triton.git@release/3.6.x#subdirectory=python/triton_kernels

setup.sh

@@ -132,15 +132,27 @@ fi
 BEST_VER=$("$BEST_PY" --version 2>&1 | awk '{print $2}')
 echo "✅ Using $BEST_PY ($BEST_VER) — compatible (≤ 3.12.x)"
 
+# Always start fresh to preserve correct install order
+rm -rf .venv
 "$BEST_PY" -m venv .venv
 source .venv/bin/activate
 run_quiet "pip upgrade" pip install --upgrade pip
 echo "   Installing unsloth-zoo + unsloth..."
-run_quiet "pip install unsloth" pip install unsloth-zoo unsloth
+run_quiet "pip install unsloth" pip install -r "$SCRIPT_DIR/requirements/base.txt"
+echo "   Installing additional unsloth dependencies..."
+run_quiet "pip install extras" pip install --no-cache-dir -r "$SCRIPT_DIR/requirements/extras.txt"
+run_quiet "pip install extras" pip install --no-deps --no-cache-dir -r "$SCRIPT_DIR/requirements/extras-no-deps.txt"
+run_quiet "pip install torchao+transformers" pip install --force-reinstall --no-cache-dir -r "$SCRIPT_DIR/requirements/overrides.txt"
+run_quiet "pip install triton_kernels" pip install --no-deps -r "$SCRIPT_DIR/requirements/triton-kernels.txt"
+# Patch: override llama_cpp.py with fix from unsloth-zoo branch
+LLAMA_CPP_DST="$(pip show unsloth-zoo | grep -i '^Location:' | awk '{print $2}')/unsloth_zoo/llama_cpp.py"
+curl -sSL "https://raw.githubusercontent.com/unslothai/unsloth-zoo/refs/heads/main/unsloth_zoo/llama_cpp.py" \
+    -o "$LLAMA_CPP_DST"
 echo "   Installing studio dependencies..."
-run_quiet "pip install extras" pip install typer fastapi uvicorn pydantic matplotlib pandas nest_asyncio "datasets==4.3.0" pyjwt easydict addict
+run_quiet "pip install studio" pip install -r "$SCRIPT_DIR/requirements/studio.txt"
 echo "✅ Python dependencies installed"
 
+
 # ── 7. Add shell alias ──
 # Note: venv activation does NOT persist across terminal sessions.
 # This alias hardcodes the venv python path so users don't need to activate.

studio/backend/requirements.txt

@@ -1,7 +0,0 @@
-fastapi>=0.100.0
-uvicorn>=0.27.0
-pydantic>=2.0
-torch
-psutil
-nest-asyncio>=1.5.8
-