feat: introduce single-env Python dependency management for streamlined compatibility
- Added constrained dependency files for single-env installations: `constraints.txt`, `data-designer.txt`, and `data-designer-deps.txt`. - Implemented a `patch_metadata.py` script to resolve metadata conflicts between dependency versions. - Updated `setup.sh` to integrate single-env setup, including dependency installation and metadata patching. - Upgraded `fastmcp` and `websockets` versions in `extras.txt` for compatibility. - Commented out unused "Start Tutorial" button in `data-recipes-page.tsx`.
This commit is contained in:
parent
dad78ae0ce
commit
c8c844a4d6
8 changed files with 150 additions and 36 deletions
47
setup.sh
47
setup.sh
|
|
@ -155,43 +155,44 @@ fi
|
|||
BEST_VER=$("$BEST_PY" --version 2>&1 | awk '{print $2}')
|
||||
echo "✅ Using $BEST_PY ($BEST_VER) — compatible (≤ 3.12.x)"
|
||||
|
||||
if [ "$IS_COLAB" = true ]; then
|
||||
# Colab: install packages directly without venv
|
||||
REQ_ROOT="$SCRIPT_DIR/studio/backend/requirements"
|
||||
SINGLE_ENV_CONSTRAINTS="$REQ_ROOT/single-env/constraints.txt"
|
||||
SINGLE_ENV_DATA_DESIGNER="$REQ_ROOT/single-env/data-designer.txt"
|
||||
SINGLE_ENV_DATA_DESIGNER_DEPS="$REQ_ROOT/single-env/data-designer-deps.txt"
|
||||
SINGLE_ENV_PATCH="$REQ_ROOT/single-env/patch_metadata.py"
|
||||
|
||||
install_python_stack() {
|
||||
run_quiet "pip upgrade" pip install --upgrade pip
|
||||
echo " Installing unsloth-zoo + unsloth..."
|
||||
run_quiet "pip install unsloth" pip install -r "$SCRIPT_DIR/studio/backend/requirements/base.txt"
|
||||
run_quiet "pip install unsloth" pip install --no-cache-dir -c "$SINGLE_ENV_CONSTRAINTS" -r "$REQ_ROOT/base.txt"
|
||||
echo " Installing additional unsloth dependencies..."
|
||||
run_quiet "pip install extras" pip install --no-cache-dir -r "$SCRIPT_DIR/studio/backend/requirements/extras.txt"
|
||||
run_quiet "pip install extras" pip install --no-deps --no-cache-dir -r "$SCRIPT_DIR/studio/backend/requirements/extras-no-deps.txt"
|
||||
run_quiet "pip install torchao+transformers" pip install --force-reinstall --no-cache-dir -r "$SCRIPT_DIR/studio/backend/requirements/overrides.txt"
|
||||
run_quiet "pip install triton_kernels" pip install --no-deps -r "$SCRIPT_DIR/studio/backend/requirements/triton-kernels.txt"
|
||||
run_quiet "pip install extras" pip install --no-cache-dir -c "$SINGLE_ENV_CONSTRAINTS" -r "$REQ_ROOT/extras.txt"
|
||||
run_quiet "pip install torchao+transformers" pip install --force-reinstall --no-cache-dir -c "$SINGLE_ENV_CONSTRAINTS" -r "$REQ_ROOT/overrides.txt"
|
||||
run_quiet "pip install triton_kernels" pip install --no-deps --no-cache-dir -r "$REQ_ROOT/triton-kernels.txt"
|
||||
# Patch: override llama_cpp.py with fix from unsloth-zoo branch
|
||||
LLAMA_CPP_DST="$(pip show unsloth-zoo | grep -i '^Location:' | awk '{print $2}')/unsloth_zoo/llama_cpp.py"
|
||||
curl -sSL "https://raw.githubusercontent.com/unslothai/unsloth-zoo/refs/heads/main/unsloth_zoo/llama_cpp.py" \
|
||||
-o "$LLAMA_CPP_DST"
|
||||
echo " Installing studio dependencies..."
|
||||
run_quiet "pip install studio" pip install -r "$SCRIPT_DIR/studio/backend/requirements/studio.txt"
|
||||
run_quiet "pip install studio" pip install --no-cache-dir -c "$SINGLE_ENV_CONSTRAINTS" -r "$REQ_ROOT/studio.txt"
|
||||
echo " Installing data-designer dependencies..."
|
||||
run_quiet "pip install data-designer deps" pip install --no-cache-dir -c "$SINGLE_ENV_CONSTRAINTS" -r "$SINGLE_ENV_DATA_DESIGNER_DEPS"
|
||||
echo " Installing data-designer..."
|
||||
run_quiet "pip install data-designer" pip install --no-cache-dir --no-deps -c "$SINGLE_ENV_CONSTRAINTS" -r "$SINGLE_ENV_DATA_DESIGNER"
|
||||
run_quiet "patch single-env metadata" python "$SINGLE_ENV_PATCH"
|
||||
run_quiet "pip check" pip check
|
||||
echo "✅ Python dependencies installed"
|
||||
}
|
||||
|
||||
if [ "$IS_COLAB" = true ]; then
|
||||
# Colab: install packages directly without venv
|
||||
install_python_stack
|
||||
else
|
||||
# Local: create venv (always start fresh to preserve correct install order)
|
||||
rm -rf .venv
|
||||
"$BEST_PY" -m venv .venv
|
||||
source .venv/bin/activate
|
||||
run_quiet "pip upgrade" pip install --upgrade pip
|
||||
echo " Installing unsloth-zoo + unsloth..."
|
||||
run_quiet "pip install unsloth" pip install -r "$SCRIPT_DIR/studio/backend/requirements/base.txt"
|
||||
echo " Installing additional unsloth dependencies..."
|
||||
run_quiet "pip install extras" pip install --no-cache-dir -r "$SCRIPT_DIR/studio/backend/requirements/extras.txt"
|
||||
run_quiet "pip install extras" pip install --no-deps --no-cache-dir -r "$SCRIPT_DIR/studio/backend/requirements/extras-no-deps.txt"
|
||||
run_quiet "pip install torchao+transformers" pip install --force-reinstall --no-cache-dir -r "$SCRIPT_DIR/studio/backend/requirements/overrides.txt"
|
||||
run_quiet "pip install triton_kernels" pip install --no-deps -r "$SCRIPT_DIR/studio/backend/requirements/triton-kernels.txt"
|
||||
# Patch: override llama_cpp.py with fix from unsloth-zoo branch
|
||||
LLAMA_CPP_DST="$(pip show unsloth-zoo | grep -i '^Location:' | awk '{print $2}')/unsloth_zoo/llama_cpp.py"
|
||||
curl -sSL "https://raw.githubusercontent.com/unslothai/unsloth-zoo/refs/heads/main/unsloth_zoo/llama_cpp.py" \
|
||||
-o "$LLAMA_CPP_DST"
|
||||
echo " Installing studio dependencies..."
|
||||
run_quiet "pip install studio" pip install -r "$SCRIPT_DIR/studio/backend/requirements/studio.txt"
|
||||
echo "✅ Python dependencies installed"
|
||||
install_python_stack
|
||||
|
||||
# ── 7. WSL: pre-install GGUF build dependencies ──
|
||||
# On WSL, sudo requires a password and can't be entered during GGUF export
|
||||
|
|
|
|||
|
|
@ -51,6 +51,6 @@ addict
|
|||
easydict
|
||||
einops
|
||||
tabulate
|
||||
fastmcp>=2.0.0
|
||||
fastmcp>=3.0.2
|
||||
openai>=2.7.2
|
||||
websockets>=13.0,<14
|
||||
websockets>=15.0.1
|
||||
|
|
|
|||
16
studio/backend/requirements/single-env/constraints.txt
Normal file
16
studio/backend/requirements/single-env/constraints.txt
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
# Single-env pins for unsloth + studio + data-designer
|
||||
# Keep compatible with unsloth transformers bounds.
|
||||
transformers==4.57.1
|
||||
trl==0.23.1
|
||||
huggingface-hub==0.36.2
|
||||
|
||||
# Studio stack
|
||||
datasets==4.3.0
|
||||
pyarrow==23.0.1
|
||||
|
||||
# FastMCP/OpenEnv compat
|
||||
fastmcp>=3.0.2
|
||||
mcp>=1.24,<2
|
||||
websockets>=15.0.1
|
||||
|
||||
pandas==2.3.3
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
# Data Designer runtime deps installed explicitly (single-env mode).
|
||||
anyascii<1,>=0.3.3
|
||||
duckdb<2,>=1.1.3
|
||||
faker<21,>=20.1.0
|
||||
httpx<1,>=0.27.2
|
||||
httpx-retries<1,>=0.4.2
|
||||
json-repair<1,>=0.48.0
|
||||
jsonpath-rust-bindings<2,>=1.0
|
||||
jsonschema<5,>=4.0.0
|
||||
litellm<1.80.12,>=1.73.6
|
||||
lxml<7,>=6.0.2
|
||||
marko<3,>=2.1.2
|
||||
networkx<4,>=3.0
|
||||
python-json-logger<4,>=3
|
||||
ruff<1,>=0.14.10
|
||||
scipy<2,>=1.11.0
|
||||
sqlfluff<4,>=3.2.0
|
||||
tiktoken<1,>=0.8.0
|
||||
5
studio/backend/requirements/single-env/data-designer.txt
Normal file
5
studio/backend/requirements/single-env/data-designer.txt
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
# Install Data Designer in same env as Unsloth.
|
||||
data-designer==0.5.1
|
||||
data-designer-config==0.5.1
|
||||
data-designer-engine==0.5.1
|
||||
prompt-toolkit>=3,<4
|
||||
74
studio/backend/requirements/single-env/patch_metadata.py
Normal file
74
studio/backend/requirements/single-env/patch_metadata.py
Normal file
|
|
@ -0,0 +1,74 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Relax strict metadata pins so pip check matches known working single-env stack.
|
||||
|
||||
Why:
|
||||
- data-designer pins huggingface-hub>=1.0.1 and pyarrow<20.
|
||||
- unsloth/transformers pins huggingface-hub<1.
|
||||
- studio datasets pins pyarrow>=21.
|
||||
|
||||
Runtime works in this app with hub 0.36.x + pyarrow 23.x, but metadata conflicts.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.metadata as im
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
TARGETS = (
|
||||
"data-designer",
|
||||
"data-designer-engine",
|
||||
"data-designer-config",
|
||||
)
|
||||
|
||||
PATCHES: tuple[tuple[re.Pattern[str], str], ...] = (
|
||||
(
|
||||
re.compile(r"^Requires-Dist: huggingface-hub<2,>=1\.0\.1$", re.MULTILINE),
|
||||
"Requires-Dist: huggingface-hub<2,>=0.34.0",
|
||||
),
|
||||
(
|
||||
re.compile(r"^Requires-Dist: pyarrow<20,>=19\.0\.1$", re.MULTILINE),
|
||||
"Requires-Dist: pyarrow>=21.0.0",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def metadata_path(dist_name: str) -> Path | None:
|
||||
try:
|
||||
dist = im.distribution(dist_name)
|
||||
except im.PackageNotFoundError:
|
||||
return None
|
||||
for f in dist.files or []:
|
||||
sf = str(f)
|
||||
if sf.endswith(".dist-info/METADATA"):
|
||||
return Path(dist.locate_file(f))
|
||||
return None
|
||||
|
||||
|
||||
def patch_file(path: Path) -> bool:
|
||||
original = path.read_text(encoding="utf-8")
|
||||
updated = original
|
||||
for pattern, repl in PATCHES:
|
||||
updated = pattern.sub(repl, updated)
|
||||
if updated == original:
|
||||
return False
|
||||
path.write_text(updated, encoding="utf-8")
|
||||
return True
|
||||
|
||||
|
||||
def main() -> int:
|
||||
changed = 0
|
||||
checked = 0
|
||||
for name in TARGETS:
|
||||
p = metadata_path(name)
|
||||
if p is None:
|
||||
continue
|
||||
checked += 1
|
||||
if patch_file(p):
|
||||
changed += 1
|
||||
print(f"single-env metadata patch: checked={checked}, changed={changed}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
|
|
@ -11,4 +11,4 @@ pyjwt
|
|||
easydict
|
||||
addict
|
||||
gradio>=4.0.0
|
||||
huggingface-hub==0.36.0
|
||||
huggingface-hub==0.36.2
|
||||
|
|
@ -408,16 +408,16 @@ export function DataRecipesPage(): ReactElement {
|
|||
</EmptyDescription>
|
||||
</EmptyHeader>
|
||||
<EmptyContent className="max-w-6xl items-stretch">
|
||||
<Button
|
||||
type="button"
|
||||
variant="secondary"
|
||||
className="mx-auto"
|
||||
onClick={() => setLearningDialogOpen(true)}
|
||||
disabled={isBusy}
|
||||
>
|
||||
<HugeiconsIcon icon={CookBookIcon} className="size-4" />
|
||||
Start Tutorial
|
||||
</Button>
|
||||
{/*<Button*/}
|
||||
{/* type="button"*/}
|
||||
{/* variant="secondary"*/}
|
||||
{/* className="mx-auto"*/}
|
||||
{/* onClick={() => setLearningDialogOpen(true)}*/}
|
||||
{/* disabled={isBusy}*/}
|
||||
{/*>*/}
|
||||
{/* <HugeiconsIcon icon={CookBookIcon} className="size-4" />*/}
|
||||
{/* Start Tutorial*/}
|
||||
{/*</Button>*/}
|
||||
<LearningRecipeCards
|
||||
onSelect={(template) => {
|
||||
openLearningRecipe(template).catch(() => undefined);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue