From fa9505439987ee23b4f4a563b2240b1771dd948b Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 28 Jul 2026 20:56:11 -0700 Subject: [PATCH 1/2] Gate the torchcodec audio extras to platforms that have a wheel (#7587) --- pyproject.toml | 11 +++-- tests/python/test_torchcodec_torch_compat.py | 47 ++++++++++++++++++++ 2 files changed, 55 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7359a51fa6..ce19d21399 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -128,14 +128,19 @@ huggingfacenotorch = [ ] # torchcodec backend for Gemma audio / datasets>=4 (#7225). # Pick the audio-torch* pin matching your torch minor (see TORCH_TORCHCODEC). +# torchcodec publishes no sdist and only manylinux_2_28_x86_64, macosx_*_arm64 +# and win_amd64 wheels, so Linux aarch64, Windows ARM64 and Intel Mac have +# nothing to resolve and pip fails the whole install rather than skipping audio. +# Gate on the platforms that have a wheel, matching +# PLATFORM_LACKS_TORCHCODEC_WHEEL in studio/install_python_stack.py. audio-torch210 = [ - "torchcodec>=0.10.0,<0.11.0 ; python_version >= '3.10'", + "torchcodec>=0.10.0,<0.11.0 ; python_version >= '3.10' and (((sys_platform == 'linux' or sys_platform == 'win32') and (platform_machine == 'x86_64' or platform_machine == 'AMD64')) or (sys_platform == 'darwin' and platform_machine == 'arm64'))", ] audio-torch290 = [ - "torchcodec>=0.8.0,<0.10.0 ; python_version >= '3.10'", + "torchcodec>=0.8.0,<0.10.0 ; python_version >= '3.10' and (((sys_platform == 'linux' or sys_platform == 'win32') and (platform_machine == 'x86_64' or platform_machine == 'AMD64')) or (sys_platform == 'darwin' and platform_machine == 'arm64'))", ] audio-torch280 = [ - "torchcodec>=0.6.0,<0.8.0 ; python_version >= '3.9'", + "torchcodec>=0.6.0,<0.8.0 ; python_version >= '3.9' and (((sys_platform == 'linux' or sys_platform == 'win32') and (platform_machine == 'x86_64' or platform_machine == 'AMD64')) or (sys_platform == 'darwin' and platform_machine == 'arm64'))", ] huggingface = [ "unsloth[huggingfacenotorch]", diff --git a/tests/python/test_torchcodec_torch_compat.py b/tests/python/test_torchcodec_torch_compat.py index 6ad16a73f4..728a51a321 100644 --- a/tests/python/test_torchcodec_torch_compat.py +++ b/tests/python/test_torchcodec_torch_compat.py @@ -11,12 +11,21 @@ import sys import types from pathlib import Path +import pytest + REPO_ROOT = Path(__file__).resolve().parents[2] PYPROJECT = REPO_ROOT / "pyproject.toml" IMPORT_FIXES_PATH = REPO_ROOT / "unsloth" / "import_fixes.py" +def _tomllib(): + if sys.version_info >= (3, 11): + import tomllib + return tomllib + return pytest.importorskip("tomli") + + def _load_import_fixes_module(): spec = importlib.util.spec_from_file_location( "unsloth_import_fixes_under_test", @@ -127,3 +136,41 @@ def test_import_fixes_loads_on_python39_syntax(): """Regression: module must import on 3.9 (postponed annotations for str | None).""" fixes = _load_import_fixes_module() assert callable(fixes._torchcodec_version_mismatch_hint) + + +def test_audio_extras_are_gated_to_platforms_with_a_torchcodec_wheel(): + """torchcodec publishes no sdist and no wheel for Linux aarch64, Windows ARM64 or + Intel Mac, so an ungated pin makes pip fail the whole install on those hosts instead + of just skipping audio -- and the cu*/rocm*/intel torch 2.10 extras pull it in. + The marker must match PLATFORM_LACKS_TORCHCODEC_WHEEL in install_python_stack.py. + """ + markers = pytest.importorskip("packaging.markers") + tomllib = _tomllib() + extras = tomllib.loads(PYPROJECT.read_text(encoding = "utf-8"))["project"][ + "optional-dependencies" + ] + audio = {n: d for n, d in extras.items() if n.startswith("audio-torch")} + assert audio, "expected audio-torch* extras" + + supported = [ + {"sys_platform": "linux", "platform_machine": "x86_64"}, + {"sys_platform": "win32", "platform_machine": "AMD64"}, + {"sys_platform": "darwin", "platform_machine": "arm64"}, + ] + unsupported = [ + {"sys_platform": "linux", "platform_machine": "aarch64"}, + {"sys_platform": "win32", "platform_machine": "ARM64"}, + {"sys_platform": "darwin", "platform_machine": "x86_64"}, + ] + for name, deps in audio.items(): + for dep in deps: + _, _, marker_text = dep.partition(";") + assert marker_text.strip(), f"{name}: {dep!r} has no marker" + marker = markers.Marker(marker_text.strip()) + env = {"python_version": "3.12"} + for case in supported: + assert marker.evaluate({**env, **case}), f"{name} must install on {case}" + for case in unsupported: + assert not marker.evaluate( + {**env, **case} + ), f"{name} has no wheel for {case} and must not be resolved there" From bc07d3a2df0bf5bca9395db259a1bd96887d3c4d Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 28 Jul 2026 21:16:31 -0700 Subject: [PATCH 2/2] Installer: wrap install.sh in a function so a piped install cannot report curl (56) (#7548) * Installer: wrap install.sh in a function so a piped install cannot report curl (56) `curl -fsSL https://unsloth.ai/install.sh | sh` makes sh the READER of a pipe. The file is ~150KB, far more than a pipe buffer holds, so a top-level `exit` left sh dead with thousands of lines unread. The write end then failed and curl appended curl: (56) Failure writing output to destination, passed 16357 returned 0 after the installer's own message, which reads as a download failure rather than the real diagnosis. 29 of the 35 exits are in the first half of the file, so every early failure on every platform looked like a bad download. Measured, piping this file into sh and forcing an early exit: before: writer rc=141 (SIGPIPE) reader rc=1 after: writer rc=0 reader rc=1 Through a real curl against a local server, curl rc went 23 -> 0 while the installer's own exit code kept propagating. Defining a function forces sh to parse to the closing brace before running anything, so the pipe is always drained. install.ps1 has always had this shape (Install-UnslothStudio invoked at the end of the file); this brings install.sh into line. Deliberately not reindented. Shell ignores leading whitespace, so the diff stays two hunks instead of 4400 reflowed lines, and `exit` still exits the shell from inside a function, so no control flow changes. tests/sh/test_install_pipe_safety.sh pins both halves of the contract: the writer must survive, and the installer's real exit code must still reach the caller. It fails against the unwrapped file (writer rc=141). * Tighten the pipe-safety comments Compress the install.sh wrapper rationale and the test header down to the parts that are not obvious from the code. Comments only, the parsed command tree of both files is byte identical. --------- Co-authored-by: danielhanchen --- install.sh | 16 +++++ tests/sh/test_install_pipe_safety.sh | 89 ++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+) create mode 100755 tests/sh/test_install_pipe_safety.sh diff --git a/install.sh b/install.sh index fc9aa0a431..166beeb52c 100755 --- a/install.sh +++ b/install.sh @@ -19,6 +19,17 @@ # SPDX-License-Identifier: AGPL-3.0-only # Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 set -e +# ── Why the installer lives in a function ── +# Under `curl ... | sh`, sh is the pipe READER. This file is ~150KB, so a top-level +# `exit` left most of it unread, the write end failed, and curl tacked +# "(56) Failure writing output to destination" onto our own error message. Wrapping +# the body forces sh to parse to the closing brace first, so the pipe always drains +# (install.ps1 has always had this shape). +# +# Body is deliberately NOT reindented: reflowing 4000+ lines would bury the change, +# and `exit` still exits the shell from inside a function. Do not add +# `exec < /dev/null`: for a piped shell that closes the script's own source. +_unsloth_main() { # ── Output style (aligned with studio/setup.sh) ── RULE="" @@ -4447,3 +4458,8 @@ else substep "(add -H 0.0.0.0 --cloudflare for a public Cloudflare HTTPS link, or --secure to keep the raw port private; anyone with the API key can run code)" echo "" fi + +} + +# Every byte above is parsed before this line runs, which is the point. +_unsloth_main "$@" diff --git a/tests/sh/test_install_pipe_safety.sh b/tests/sh/test_install_pipe_safety.sh new file mode 100755 index 0000000000..be479dd5a5 --- /dev/null +++ b/tests/sh/test_install_pipe_safety.sh @@ -0,0 +1,89 @@ +#!/bin/bash +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 +# +# Guards that `curl ... | sh` cannot report a bogus transport error. +# +# History: install.sh was ~150KB of top-level statements. A top-level `exit` left most +# of the file unread, the write end failed, and curl appended "(56) Failure writing +# output to destination" (or "(23) Failed writing body") after our own error message, +# so users read a real diagnosis as a broken download. The fix is structural: the body +# lives in _unsloth_main, so sh parses the whole file before running anything. +# +# This pins both halves of that contract: the writer must not be killed, AND the +# installer's own exit code must still reach the caller. +set -e + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +INSTALL_SH="$SCRIPT_DIR/../../install.sh" +PASS=0 +FAIL=0 + +assert_eq() { + _label="$1"; _expected="$2"; _actual="$3" + if [ "$_actual" = "$_expected" ]; then + echo " PASS: $_label" + PASS=$((PASS + 1)) + else + echo " FAIL: $_label (expected '$_expected', got '$_actual')" + FAIL=$((FAIL + 1)) + fi +} + +echo "=== structure ===" + +# The wrapper must be invoked on the LAST executable line, or sh starts executing +# before it has drained the pipe. +if grep -q '^_unsloth_main() {' "$INSTALL_SH"; then + echo " PASS: _unsloth_main is defined at top level" + PASS=$((PASS + 1)) +else + echo " FAIL: install.sh is not wrapped in _unsloth_main -- curl-pipe safety is gone" + FAIL=$((FAIL + 1)) +fi + +_last="$(grep -vE '^\s*(#|$)' "$INSTALL_SH" | tail -1)" +assert_eq "last statement invokes the wrapper" '_unsloth_main "$@"' "$_last" + +# Below one pipe buffer the file would fit in the kernel's buffer and this test would +# prove nothing, so fail loudly instead of passing vacuously. +_bytes="$(wc -c < "$INSTALL_SH" | tr -d ' ')" +if [ "$_bytes" -gt 65536 ]; then + echo " PASS: install.sh ($_bytes bytes) exceeds a 64KiB pipe buffer, so this matters" + PASS=$((PASS + 1)) +else + echo " FAIL: install.sh is only $_bytes bytes; re-derive whether pipe safety still applies" + FAIL=$((FAIL + 1)) +fi + +echo "=== behaviour: an early exit must not kill the writer ===" + +# `--python` with no argument exits 1 from argument validation having done no work: no +# venv, no downloads, no filesystem writes. Deterministic and safe to run for real. +# +# PIPESTATUS must be read on the very next line, so drop errexit around the pipeline +# rather than appending `|| true`, which would clobber it with the status of `true`. +set +e +cat "$INSTALL_SH" | sh -s -- --python >/dev/null 2>&1 +_pipe=("${PIPESTATUS[@]}") +set -e +_writer_rc="${_pipe[0]}" +_reader_rc="${_pipe[1]}" + +# A writer rc of 141 (128 + SIGPIPE) is the failure mode curl reports as (56)/(23). +assert_eq "writer survives the early exit (not SIGPIPE)" "0" "$_writer_rc" +assert_eq "installer's own exit code still propagates" "1" "$_reader_rc" + +echo "=== behaviour: the same holds for a mid-file exit ===" +# `--package '-evil'` exits from a later validation block, still before any filesystem +# work, so the property is not specific to one early branch. +set +e +cat "$INSTALL_SH" | sh -s -- --package '-evil' >/dev/null 2>&1 +_pipe2=("${PIPESTATUS[@]}") +set -e +assert_eq "writer survives a later exit" "0" "${_pipe2[0]}" +assert_eq "later exit code propagates" "1" "${_pipe2[1]}" + +echo "" +echo "=== $PASS passed, $FAIL failed ===" +[ "$FAIL" -eq 0 ] || exit 1