diff --git a/setup.sh b/setup.sh index 1f55e141fd..13b8f42326 100755 --- a/setup.sh +++ b/setup.sh @@ -155,43 +155,44 @@ fi BEST_VER=$("$BEST_PY" --version 2>&1 | awk '{print $2}') echo "✅ Using $BEST_PY ($BEST_VER) — compatible (≤ 3.12.x)" -if [ "$IS_COLAB" = true ]; then - # Colab: install packages directly without venv +REQ_ROOT="$SCRIPT_DIR/studio/backend/requirements" +SINGLE_ENV_CONSTRAINTS="$REQ_ROOT/single-env/constraints.txt" +SINGLE_ENV_DATA_DESIGNER="$REQ_ROOT/single-env/data-designer.txt" +SINGLE_ENV_DATA_DESIGNER_DEPS="$REQ_ROOT/single-env/data-designer-deps.txt" +SINGLE_ENV_PATCH="$REQ_ROOT/single-env/patch_metadata.py" + +install_python_stack() { run_quiet "pip upgrade" pip install --upgrade pip echo " Installing unsloth-zoo + unsloth..." - run_quiet "pip install unsloth" pip install -r "$SCRIPT_DIR/studio/backend/requirements/base.txt" + run_quiet "pip install unsloth" pip install --no-cache-dir -c "$SINGLE_ENV_CONSTRAINTS" -r "$REQ_ROOT/base.txt" echo " Installing additional unsloth dependencies..." - run_quiet "pip install extras" pip install --no-cache-dir -r "$SCRIPT_DIR/studio/backend/requirements/extras.txt" - run_quiet "pip install extras" pip install --no-deps --no-cache-dir -r "$SCRIPT_DIR/studio/backend/requirements/extras-no-deps.txt" - run_quiet "pip install torchao+transformers" pip install --force-reinstall --no-cache-dir -r "$SCRIPT_DIR/studio/backend/requirements/overrides.txt" - run_quiet "pip install triton_kernels" pip install --no-deps -r "$SCRIPT_DIR/studio/backend/requirements/triton-kernels.txt" + run_quiet "pip install extras" pip install --no-cache-dir -c "$SINGLE_ENV_CONSTRAINTS" -r "$REQ_ROOT/extras.txt" + run_quiet "pip install torchao+transformers" pip install --force-reinstall --no-cache-dir -c "$SINGLE_ENV_CONSTRAINTS" -r "$REQ_ROOT/overrides.txt" + run_quiet "pip install triton_kernels" pip install --no-deps --no-cache-dir -r "$REQ_ROOT/triton-kernels.txt" # Patch: override llama_cpp.py with fix from unsloth-zoo branch LLAMA_CPP_DST="$(pip show unsloth-zoo | grep -i '^Location:' | awk '{print $2}')/unsloth_zoo/llama_cpp.py" curl -sSL "https://raw.githubusercontent.com/unslothai/unsloth-zoo/refs/heads/main/unsloth_zoo/llama_cpp.py" \ -o "$LLAMA_CPP_DST" echo " Installing studio dependencies..." - run_quiet "pip install studio" pip install -r "$SCRIPT_DIR/studio/backend/requirements/studio.txt" + run_quiet "pip install studio" pip install --no-cache-dir -c "$SINGLE_ENV_CONSTRAINTS" -r "$REQ_ROOT/studio.txt" + echo " Installing data-designer dependencies..." + run_quiet "pip install data-designer deps" pip install --no-cache-dir -c "$SINGLE_ENV_CONSTRAINTS" -r "$SINGLE_ENV_DATA_DESIGNER_DEPS" + echo " Installing data-designer..." + run_quiet "pip install data-designer" pip install --no-cache-dir --no-deps -c "$SINGLE_ENV_CONSTRAINTS" -r "$SINGLE_ENV_DATA_DESIGNER" + run_quiet "patch single-env metadata" python "$SINGLE_ENV_PATCH" + run_quiet "pip check" pip check echo "✅ Python dependencies installed" +} + +if [ "$IS_COLAB" = true ]; then + # Colab: install packages directly without venv + install_python_stack else # Local: create venv (always start fresh to preserve correct install order) rm -rf .venv "$BEST_PY" -m venv .venv source .venv/bin/activate - run_quiet "pip upgrade" pip install --upgrade pip - echo " Installing unsloth-zoo + unsloth..." - run_quiet "pip install unsloth" pip install -r "$SCRIPT_DIR/studio/backend/requirements/base.txt" - echo " Installing additional unsloth dependencies..." - run_quiet "pip install extras" pip install --no-cache-dir -r "$SCRIPT_DIR/studio/backend/requirements/extras.txt" - run_quiet "pip install extras" pip install --no-deps --no-cache-dir -r "$SCRIPT_DIR/studio/backend/requirements/extras-no-deps.txt" - run_quiet "pip install torchao+transformers" pip install --force-reinstall --no-cache-dir -r "$SCRIPT_DIR/studio/backend/requirements/overrides.txt" - run_quiet "pip install triton_kernels" pip install --no-deps -r "$SCRIPT_DIR/studio/backend/requirements/triton-kernels.txt" - # Patch: override llama_cpp.py with fix from unsloth-zoo branch - LLAMA_CPP_DST="$(pip show unsloth-zoo | grep -i '^Location:' | awk '{print $2}')/unsloth_zoo/llama_cpp.py" - curl -sSL "https://raw.githubusercontent.com/unslothai/unsloth-zoo/refs/heads/main/unsloth_zoo/llama_cpp.py" \ - -o "$LLAMA_CPP_DST" - echo " Installing studio dependencies..." - run_quiet "pip install studio" pip install -r "$SCRIPT_DIR/studio/backend/requirements/studio.txt" - echo "✅ Python dependencies installed" + install_python_stack # ── 7. WSL: pre-install GGUF build dependencies ── # On WSL, sudo requires a password and can't be entered during GGUF export diff --git a/studio/backend/requirements/extras.txt b/studio/backend/requirements/extras.txt index 3ed20faa8b..51ef69cf5d 100644 --- a/studio/backend/requirements/extras.txt +++ b/studio/backend/requirements/extras.txt @@ -51,6 +51,6 @@ addict easydict einops tabulate -fastmcp>=2.0.0 +fastmcp>=3.0.2 openai>=2.7.2 -websockets>=13.0,<14 +websockets>=15.0.1 diff --git a/studio/backend/requirements/single-env/constraints.txt b/studio/backend/requirements/single-env/constraints.txt new file mode 100644 index 0000000000..1789bbf713 --- /dev/null +++ b/studio/backend/requirements/single-env/constraints.txt @@ -0,0 +1,16 @@ +# Single-env pins for unsloth + studio + data-designer +# Keep compatible with unsloth transformers bounds. +transformers==4.57.1 +trl==0.23.1 +huggingface-hub==0.36.2 + +# Studio stack +datasets==4.3.0 +pyarrow==23.0.1 + +# FastMCP/OpenEnv compat +fastmcp>=3.0.2 +mcp>=1.24,<2 +websockets>=15.0.1 + +pandas==2.3.3 diff --git a/studio/backend/requirements/single-env/data-designer-deps.txt b/studio/backend/requirements/single-env/data-designer-deps.txt new file mode 100644 index 0000000000..cbf8856073 --- /dev/null +++ b/studio/backend/requirements/single-env/data-designer-deps.txt @@ -0,0 +1,18 @@ +# Data Designer runtime deps installed explicitly (single-env mode). +anyascii<1,>=0.3.3 +duckdb<2,>=1.1.3 +faker<21,>=20.1.0 +httpx<1,>=0.27.2 +httpx-retries<1,>=0.4.2 +json-repair<1,>=0.48.0 +jsonpath-rust-bindings<2,>=1.0 +jsonschema<5,>=4.0.0 +litellm<1.80.12,>=1.73.6 +lxml<7,>=6.0.2 +marko<3,>=2.1.2 +networkx<4,>=3.0 +python-json-logger<4,>=3 +ruff<1,>=0.14.10 +scipy<2,>=1.11.0 +sqlfluff<4,>=3.2.0 +tiktoken<1,>=0.8.0 diff --git a/studio/backend/requirements/single-env/data-designer.txt b/studio/backend/requirements/single-env/data-designer.txt new file mode 100644 index 0000000000..c5ddcddc36 --- /dev/null +++ b/studio/backend/requirements/single-env/data-designer.txt @@ -0,0 +1,5 @@ +# Install Data Designer in same env as Unsloth. +data-designer==0.5.1 +data-designer-config==0.5.1 +data-designer-engine==0.5.1 +prompt-toolkit>=3,<4 diff --git a/studio/backend/requirements/single-env/patch_metadata.py b/studio/backend/requirements/single-env/patch_metadata.py new file mode 100644 index 0000000000..b579637463 --- /dev/null +++ b/studio/backend/requirements/single-env/patch_metadata.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +"""Relax strict metadata pins so pip check matches known working single-env stack. + +Why: +- data-designer pins huggingface-hub>=1.0.1 and pyarrow<20. +- unsloth/transformers pins huggingface-hub<1. +- studio datasets pins pyarrow>=21. + +Runtime works in this app with hub 0.36.x + pyarrow 23.x, but metadata conflicts. +""" + +from __future__ import annotations + +import importlib.metadata as im +import re +from pathlib import Path + +TARGETS = ( + "data-designer", + "data-designer-engine", + "data-designer-config", +) + +PATCHES: tuple[tuple[re.Pattern[str], str], ...] = ( + ( + re.compile(r"^Requires-Dist: huggingface-hub<2,>=1\.0\.1$", re.MULTILINE), + "Requires-Dist: huggingface-hub<2,>=0.34.0", + ), + ( + re.compile(r"^Requires-Dist: pyarrow<20,>=19\.0\.1$", re.MULTILINE), + "Requires-Dist: pyarrow>=21.0.0", + ), +) + + +def metadata_path(dist_name: str) -> Path | None: + try: + dist = im.distribution(dist_name) + except im.PackageNotFoundError: + return None + for f in dist.files or []: + sf = str(f) + if sf.endswith(".dist-info/METADATA"): + return Path(dist.locate_file(f)) + return None + + +def patch_file(path: Path) -> bool: + original = path.read_text(encoding="utf-8") + updated = original + for pattern, repl in PATCHES: + updated = pattern.sub(repl, updated) + if updated == original: + return False + path.write_text(updated, encoding="utf-8") + return True + + +def main() -> int: + changed = 0 + checked = 0 + for name in TARGETS: + p = metadata_path(name) + if p is None: + continue + checked += 1 + if patch_file(p): + changed += 1 + print(f"single-env metadata patch: checked={checked}, changed={changed}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/studio/backend/requirements/studio.txt b/studio/backend/requirements/studio.txt index 6a732664d2..244ed4f48c 100644 --- a/studio/backend/requirements/studio.txt +++ b/studio/backend/requirements/studio.txt @@ -11,4 +11,4 @@ pyjwt easydict addict gradio>=4.0.0 -huggingface-hub==0.36.0 \ No newline at end of file +huggingface-hub==0.36.2 \ No newline at end of file diff --git a/studio/frontend/src/features/data-recipes/pages/data-recipes-page.tsx b/studio/frontend/src/features/data-recipes/pages/data-recipes-page.tsx index 2eb777fed4..4481f5a0c0 100644 --- a/studio/frontend/src/features/data-recipes/pages/data-recipes-page.tsx +++ b/studio/frontend/src/features/data-recipes/pages/data-recipes-page.tsx @@ -408,16 +408,16 @@ export function DataRecipesPage(): ReactElement { - + {/* setLearningDialogOpen(true)}*/} + {/* disabled={isBusy}*/} + {/*>*/} + {/* */} + {/* Start Tutorial*/} + {/**/} { openLearningRecipe(template).catch(() => undefined);