Compare commits
4 commits
main
...
fix/torchc
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6df6f82557 | ||
|
|
8ca1764e46 | ||
|
|
fdaefa07f8 | ||
|
|
ef240d5a61 |
7 changed files with 422 additions and 19 deletions
16
.github/workflows/security-audit.yml
vendored
16
.github/workflows/security-audit.yml
vendored
|
|
@ -229,9 +229,13 @@ jobs:
|
||||||
d = tomllib.load(f)
|
d = tomllib.load(f)
|
||||||
core = d["project"]["dependencies"]
|
core = d["project"]["dependencies"]
|
||||||
extras = d["project"]["optional-dependencies"]["huggingfacenotorch"]
|
extras = d["project"]["optional-dependencies"]["huggingfacenotorch"]
|
||||||
|
# torchcodec is no longer pinned in extras-no-deps.txt (the installer
|
||||||
|
# picks the line matching the venv's torch minor), so pull the newest
|
||||||
|
# audio extra in to keep torchcodec inside the audited set.
|
||||||
|
audio = d["project"]["optional-dependencies"]["audio-torch211"]
|
||||||
print("# Auto-generated from pyproject.toml by security-audit.yml.")
|
print("# Auto-generated from pyproject.toml by security-audit.yml.")
|
||||||
print("# core deps + huggingfacenotorch extras.")
|
print("# core deps + huggingfacenotorch extras + audio-torch211.")
|
||||||
for spec in core + extras:
|
for spec in core + extras + audio:
|
||||||
print(spec)
|
print(spec)
|
||||||
PY
|
PY
|
||||||
for f in studio.txt extras.txt extras-no-deps.txt \
|
for f in studio.txt extras.txt extras-no-deps.txt \
|
||||||
|
|
@ -822,9 +826,13 @@ jobs:
|
||||||
d = tomllib.load(f)
|
d = tomllib.load(f)
|
||||||
core = d["project"]["dependencies"]
|
core = d["project"]["dependencies"]
|
||||||
extras = d["project"]["optional-dependencies"]["huggingfacenotorch"]
|
extras = d["project"]["optional-dependencies"]["huggingfacenotorch"]
|
||||||
|
# torchcodec is no longer pinned in extras-no-deps.txt (the installer
|
||||||
|
# picks the line matching the venv's torch minor), so pull the newest
|
||||||
|
# audio extra in to keep torchcodec inside the audited set.
|
||||||
|
audio = d["project"]["optional-dependencies"]["audio-torch211"]
|
||||||
print("# Auto-generated from pyproject.toml by security-audit.yml.")
|
print("# Auto-generated from pyproject.toml by security-audit.yml.")
|
||||||
print("# core deps + huggingfacenotorch extras.")
|
print("# core deps + huggingfacenotorch extras + audio-torch211.")
|
||||||
for spec in core + extras:
|
for spec in core + extras + audio:
|
||||||
print(spec)
|
print(spec)
|
||||||
PY
|
PY
|
||||||
for f in studio.txt extras.txt extras-no-deps.txt \
|
for f in studio.txt extras.txt extras-no-deps.txt \
|
||||||
|
|
|
||||||
|
|
@ -95,6 +95,9 @@ huggingfacenotorch = [
|
||||||
]
|
]
|
||||||
# torchcodec backend for Gemma audio / datasets>=4 (#7225).
|
# torchcodec backend for Gemma audio / datasets>=4 (#7225).
|
||||||
# Pick the audio-torch* pin matching your torch minor (see TORCH_TORCHCODEC).
|
# Pick the audio-torch* pin matching your torch minor (see TORCH_TORCHCODEC).
|
||||||
|
audio-torch211 = [
|
||||||
|
"torchcodec>=0.11.0,<0.12.0 ; python_version >= '3.10'",
|
||||||
|
]
|
||||||
audio-torch210 = [
|
audio-torch210 = [
|
||||||
"torchcodec>=0.10.0,<0.11.0 ; python_version >= '3.10'",
|
"torchcodec>=0.10.0,<0.11.0 ; python_version >= '3.10'",
|
||||||
]
|
]
|
||||||
|
|
|
||||||
|
|
@ -93,7 +93,10 @@ COLAB_ORACLE_BASE_URL = "https://raw.githubusercontent.com/googlecolab/backend-i
|
||||||
|
|
||||||
# torch.minor -> set of compatible torchcodec.minor strings.
|
# torch.minor -> set of compatible torchcodec.minor strings.
|
||||||
# Source: pytorch/torchcodec compatibility matrix on its README.
|
# Source: pytorch/torchcodec compatibility matrix on its README.
|
||||||
|
# Mirrors unsloth/import_fixes.py::_TORCH_TORCHCODEC_MINORS (asserted equal by
|
||||||
|
# tests/python/test_torchcodec_torch_compat.py).
|
||||||
TORCH_TORCHCODEC: dict[str, set[str]] = {
|
TORCH_TORCHCODEC: dict[str, set[str]] = {
|
||||||
|
"2.11": {"0.11"},
|
||||||
"2.10": {"0.10"},
|
"2.10": {"0.10"},
|
||||||
"2.9": {"0.8", "0.9"},
|
"2.9": {"0.8", "0.9"},
|
||||||
"2.8": {"0.6", "0.7"},
|
"2.8": {"0.6", "0.7"},
|
||||||
|
|
@ -102,6 +105,12 @@ TORCH_TORCHCODEC: dict[str, set[str]] = {
|
||||||
"2.5": {"0.1", "0.2"},
|
"2.5": {"0.1", "0.2"},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# torchcodec 0.12+ is ABI-stable against torch >= 2.11, so that half of the
|
||||||
|
# matrix is open-ended and cannot be written as a finite set of minors. Mirrors
|
||||||
|
# unsloth/import_fixes.py::_TORCHCODEC_ABI_STABLE_{TORCH,CODEC}.
|
||||||
|
TORCHCODEC_ABI_STABLE_TORCH = "2.11"
|
||||||
|
TORCHCODEC_ABI_STABLE_CODEC = "0.12"
|
||||||
|
|
||||||
# When peft >= trigger is on the resolved set, torchao >= floor must also be.
|
# When peft >= trigger is on the resolved set, torchao >= floor must also be.
|
||||||
PEFT_TORCHAO_FLOOR: list[dict[str, str]] = [
|
PEFT_TORCHAO_FLOOR: list[dict[str, str]] = [
|
||||||
{"trigger_peft": "0.19", "torchao_floor": "0.16.0"},
|
{"trigger_peft": "0.19", "torchao_floor": "0.16.0"},
|
||||||
|
|
@ -612,11 +621,31 @@ def rule_inst_004_torchcodec_torch(
|
||||||
codec_v = res.get("torchcodec")
|
codec_v = res.get("torchcodec")
|
||||||
if not torch_v or not codec_v:
|
if not torch_v or not codec_v:
|
||||||
return findings
|
return findings
|
||||||
|
if (
|
||||||
|
cmp_versions(torch_v, TORCHCODEC_ABI_STABLE_TORCH) >= 0
|
||||||
|
and cmp_versions(codec_v, TORCHCODEC_ABI_STABLE_CODEC) >= 0
|
||||||
|
):
|
||||||
|
return findings # ABI-stable pairing, not locked to one torch minor
|
||||||
t_minor = version_minor(torch_v)
|
t_minor = version_minor(torch_v)
|
||||||
c_minor = version_minor(codec_v)
|
c_minor = version_minor(codec_v)
|
||||||
allowed = TORCH_TORCHCODEC.get(t_minor)
|
allowed = TORCH_TORCHCODEC.get(t_minor)
|
||||||
if allowed is None:
|
if allowed is None:
|
||||||
return findings # unknown torch minor — don't flag
|
if cmp_versions(torch_v, TORCHCODEC_ABI_STABLE_TORCH) < 0:
|
||||||
|
return findings # torch older than the table — don't flag
|
||||||
|
# Torch at or past the ABI-stable floor with a pre-0.12 torchcodec: the
|
||||||
|
# ABI-stable branch above already returned for 0.12+, so this pin is a
|
||||||
|
# legacy build locked to an older torch minor and cannot load.
|
||||||
|
findings.append(
|
||||||
|
Finding(
|
||||||
|
rule = "R-INST-004",
|
||||||
|
file = file,
|
||||||
|
cell = cell_idx,
|
||||||
|
severity = "error",
|
||||||
|
message = f"torch=={torch_v} (minor {t_minor}) is incompatible with torchcodec=={codec_v} (minor {c_minor}); torchcodec <{TORCHCODEC_ABI_STABLE_CODEC} is built against a single older torch minor",
|
||||||
|
hint = f"pin `torchcodec>={TORCHCODEC_ABI_STABLE_CODEC}.0` (the ABI-stable line, which targets torch >={TORCHCODEC_ABI_STABLE_TORCH})",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return findings
|
||||||
if c_minor not in allowed:
|
if c_minor not in allowed:
|
||||||
findings.append(
|
findings.append(
|
||||||
Finding(
|
Finding(
|
||||||
|
|
|
||||||
|
|
@ -2,7 +2,10 @@
|
||||||
descript-audio-codec
|
descript-audio-codec
|
||||||
descript-audiotools
|
descript-audiotools
|
||||||
julius
|
julius
|
||||||
torchcodec==0.10.0
|
# torchcodec is NOT pinned here: its wheels are torch-ABI specific and a
|
||||||
|
# requirements file cannot branch on the installed torch minor (environment
|
||||||
|
# markers cannot see it). install_python_stack._select_torchcodec_spec picks
|
||||||
|
# the matching line right after this file is installed.
|
||||||
snac
|
snac
|
||||||
|
|
||||||
# peft 0.19.0 causes export subprocess shutdown issues in Unsloth;
|
# peft 0.19.0 causes export subprocess shutdown issues in Unsloth;
|
||||||
|
|
|
||||||
|
|
@ -298,6 +298,59 @@ def _select_torchao_spec(torch_version: str | None) -> str:
|
||||||
return _TORCHAO_DEFAULT_SPEC
|
return _TORCHAO_DEFAULT_SPEC
|
||||||
|
|
||||||
|
|
||||||
|
# torchcodec wheels up to 0.11 are built against exactly one torch minor and
|
||||||
|
# declare no `Requires-Dist: torch`, so pip cannot detect a mismatch and a flat
|
||||||
|
# pin in extras-no-deps.txt cannot serve every host: install.sh resolves torch
|
||||||
|
# 2.11.0 on the CUDA indexes but keeps a <2.11 default on other routes. These
|
||||||
|
# specs mirror pyproject's audio-torch2xx extras and unsloth/import_fixes.py's
|
||||||
|
# _TORCH_TORCHCODEC_MINORS (upstream's torch <-> torchcodec matrix).
|
||||||
|
#
|
||||||
|
# 0.12 onwards is ABI-stable against torch >=2.11 (torchcodec's CMakeLists sets
|
||||||
|
# TORCH_TARGET_VERSION 2.11), so torch 2.12+ takes an open-ended floor instead of
|
||||||
|
# a per-minor pin. Torch 2.11 itself keeps the 0.11 line: 0.12 dropped CUDA 12.8,
|
||||||
|
# so the cu128 index install.sh resolves torch 2.11.0 from stops at torchcodec
|
||||||
|
# 0.11.1, and a 0.12+ wheel off PyPI is built for CUDA 13.
|
||||||
|
_TORCHCODEC_DEFAULT_SPEC = "torchcodec>=0.10.0,<0.11.0"
|
||||||
|
_TORCHCODEC_ABI_STABLE_SPEC = "torchcodec>=0.12.0"
|
||||||
|
_TORCHCODEC_TORCH_SPECS: dict[int, str] = {
|
||||||
|
12: _TORCHCODEC_ABI_STABLE_SPEC,
|
||||||
|
11: "torchcodec>=0.11.0,<0.12.0",
|
||||||
|
10: "torchcodec>=0.10.0,<0.11.0",
|
||||||
|
9: "torchcodec>=0.8.0,<0.10.0",
|
||||||
|
8: "torchcodec>=0.6.0,<0.8.0",
|
||||||
|
7: "torchcodec>=0.3.0,<0.6.0",
|
||||||
|
6: "torchcodec>=0.2.0,<0.4.0",
|
||||||
|
5: "torchcodec>=0.1.0,<0.3.0",
|
||||||
|
}
|
||||||
|
_TORCHCODEC_MAX_KNOWN_MINOR = max(_TORCHCODEC_TORCH_SPECS)
|
||||||
|
|
||||||
|
|
||||||
|
def _select_torchcodec_spec(torch_version: "str | None") -> str:
|
||||||
|
"""Map an installed torch version string (e.g. '2.11.0+cu128') to the torchcodec
|
||||||
|
pip spec built against it. Falls back to _TORCHCODEC_DEFAULT_SPEC for torch <=2.4,
|
||||||
|
a non-2.x major, or an unparseable/missing version. Pure function.
|
||||||
|
"""
|
||||||
|
if not torch_version:
|
||||||
|
return _TORCHCODEC_DEFAULT_SPEC
|
||||||
|
release = str(torch_version).split("+", 1)[0] # drop +cu128/+rocm7.2/+cpu
|
||||||
|
parts = release.split(".")
|
||||||
|
try:
|
||||||
|
# Strip any pre-release/dev suffix from the minor (e.g. '11rc1' -> '11'),
|
||||||
|
# matching _select_torchao_spec.
|
||||||
|
minor_str = re.sub(r"[^0-9].*", "", parts[1]) if len(parts) > 1 else ""
|
||||||
|
major, minor = int(parts[0]), int(minor_str)
|
||||||
|
except (IndexError, ValueError):
|
||||||
|
return _TORCHCODEC_DEFAULT_SPEC
|
||||||
|
if major != 2:
|
||||||
|
return _TORCHCODEC_DEFAULT_SPEC
|
||||||
|
# Newer torch than we have a row for lands on the ABI-stable floor, which is
|
||||||
|
# forward compatible by construction (0.12+ target torch >=2.11). Never clamp
|
||||||
|
# onto the 0.11 row: that is the one release locked to torch 2.11 exactly, and
|
||||||
|
# it has no wheel on the newer CUDA indexes.
|
||||||
|
minor = min(minor, _TORCHCODEC_MAX_KNOWN_MINOR)
|
||||||
|
return _TORCHCODEC_TORCH_SPECS.get(minor, _TORCHCODEC_DEFAULT_SPEC)
|
||||||
|
|
||||||
|
|
||||||
def _probe_installed_torch_version() -> str | None:
|
def _probe_installed_torch_version() -> str | None:
|
||||||
"""Return torch.__version__ from the target venv (sys.executable), or None if
|
"""Return torch.__version__ from the target venv (sys.executable), or None if
|
||||||
torch is absent/unimportable. Cross-platform (unlike probe_torch_wheel_env,
|
torch is absent/unimportable. Cross-platform (unlike probe_torch_wheel_env,
|
||||||
|
|
@ -2766,7 +2819,10 @@ def pip_install(
|
||||||
# wheel. `unsloth studio update --local` does not pass
|
# wheel. `unsloth studio update --local` does not pass
|
||||||
# --no-torch, so the NO_TORCH filter above does not fire; do
|
# --no-torch, so the NO_TORCH filter above does not fire; do
|
||||||
# the targeted skip independently so the audio extras step
|
# the targeted skip independently so the audio extras step
|
||||||
# does not take down the whole update.
|
# does not take down the whole update. The primary guard now
|
||||||
|
# lives on the dedicated torchcodec step, which no requirements
|
||||||
|
# file feeds; this stays as belt and braces for any file that
|
||||||
|
# reintroduces a torchcodec line.
|
||||||
actual_req = _filter_requirements(actual_req, {"torchcodec"})
|
actual_req = _filter_requirements(actual_req, {"torchcodec"})
|
||||||
temp_reqs.append(actual_req)
|
temp_reqs.append(actual_req)
|
||||||
req_args_pip: list[str] = []
|
req_args_pip: list[str] = []
|
||||||
|
|
@ -2847,7 +2903,9 @@ def install_python_stack() -> int:
|
||||||
package_name = os.environ.get("STUDIO_PACKAGE_NAME", "unsloth")
|
package_name = os.environ.get("STUDIO_PACKAGE_NAME", "unsloth")
|
||||||
# --local overlays a local repo checkout after updating deps.
|
# --local overlays a local repo checkout after updating deps.
|
||||||
local_repo = os.environ.get("STUDIO_LOCAL_REPO", "")
|
local_repo = os.environ.get("STUDIO_LOCAL_REPO", "")
|
||||||
base_total = 11 if IS_WINDOWS else 12 # +1 for the anyio repair check (step 8b)
|
# +1 for the anyio repair check (step 8b), +1 for torchcodec (step 13b, which
|
||||||
|
# reports progress on every branch including its skips).
|
||||||
|
base_total = 12 if IS_WINDOWS else 13
|
||||||
if IS_MACOS:
|
if IS_MACOS:
|
||||||
base_total -= 1 # triton step is skipped on macOS
|
base_total -= 1 # triton step is skipped on macOS
|
||||||
if not IS_MACOS and not NO_TORCH:
|
if not IS_MACOS and not NO_TORCH:
|
||||||
|
|
@ -3226,6 +3284,31 @@ def install_python_stack() -> int:
|
||||||
_ensure_rocm_torch()
|
_ensure_rocm_torch()
|
||||||
_ensure_cpu_torch()
|
_ensure_cpu_torch()
|
||||||
|
|
||||||
|
# 13b. torchcodec -- pinned to the line built against the venv's torch (see
|
||||||
|
# _select_torchcodec_spec), so it must run *after* the repair above:
|
||||||
|
# that repair can move torch onto another minor (cu128 resolves 2.11.0),
|
||||||
|
# and a torchcodec picked before it would be stale again. It cannot live
|
||||||
|
# in extras-no-deps.txt because pip environment markers cannot branch on
|
||||||
|
# the installed torch version. Skipped on the same platforms pip_install
|
||||||
|
# filtered it out for (no torch at all, or no published wheel).
|
||||||
|
if NO_TORCH:
|
||||||
|
_progress("torchcodec (skipped, no torch)")
|
||||||
|
elif PLATFORM_LACKS_TORCHCODEC_WHEEL:
|
||||||
|
_progress("torchcodec (skipped, no wheel for this platform)")
|
||||||
|
else:
|
||||||
|
_progress("torchcodec")
|
||||||
|
_codec_torch_ver = _probe_installed_torch_version()
|
||||||
|
_codec_spec = _select_torchcodec_spec(_codec_torch_ver)
|
||||||
|
_safe_print(
|
||||||
|
f" torch {_codec_torch_ver or 'unknown'} detected -- installing {_codec_spec}"
|
||||||
|
)
|
||||||
|
pip_install(
|
||||||
|
"Installing torchcodec",
|
||||||
|
"--no-deps",
|
||||||
|
"--no-cache-dir",
|
||||||
|
_codec_spec,
|
||||||
|
)
|
||||||
|
|
||||||
# 14. Final check (silent; third-party conflicts are expected)
|
# 14. Final check (silent; third-party conflicts are expected)
|
||||||
subprocess.run(
|
subprocess.run(
|
||||||
[sys.executable, "-m", "pip", "check"],
|
[sys.executable, "-m", "pip", "check"],
|
||||||
|
|
|
||||||
|
|
@ -15,6 +15,7 @@ from pathlib import Path
|
||||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||||
PYPROJECT = REPO_ROOT / "pyproject.toml"
|
PYPROJECT = REPO_ROOT / "pyproject.toml"
|
||||||
IMPORT_FIXES_PATH = REPO_ROOT / "unsloth" / "import_fixes.py"
|
IMPORT_FIXES_PATH = REPO_ROOT / "unsloth" / "import_fixes.py"
|
||||||
|
EXTRAS_NO_DEPS_TXT = REPO_ROOT / "studio" / "backend" / "requirements" / "extras-no-deps.txt"
|
||||||
|
|
||||||
|
|
||||||
def _load_import_fixes_module():
|
def _load_import_fixes_module():
|
||||||
|
|
@ -127,3 +128,239 @@ def test_import_fixes_loads_on_python39_syntax():
|
||||||
"""Regression: module must import on 3.9 (postponed annotations for str | None)."""
|
"""Regression: module must import on 3.9 (postponed annotations for str | None)."""
|
||||||
fixes = _load_import_fixes_module()
|
fixes = _load_import_fixes_module()
|
||||||
assert callable(fixes._torchcodec_version_mismatch_hint)
|
assert callable(fixes._torchcodec_version_mismatch_hint)
|
||||||
|
|
||||||
|
|
||||||
|
# ── torch 2.11 (unslothai/unsloth#7225 follow-up) ──────────────────────
|
||||||
|
#
|
||||||
|
# install.sh's CUDA branch resolves torch 2.11.0 (the cu12x/cu13x indexes
|
||||||
|
# top out there), and torchcodec declares no `torch` dependency, so pip
|
||||||
|
# cannot catch a stale 0.10 pairing. These lock the 2.11 row in.
|
||||||
|
|
||||||
|
|
||||||
|
def _load_notebook_validator_module():
|
||||||
|
"""Load by path: `from scripts import ...` picks up whatever `scripts`
|
||||||
|
package happens to be on sys.path first, which is not always this repo's."""
|
||||||
|
spec = importlib.util.spec_from_file_location(
|
||||||
|
"unsloth_notebook_validator_under_test",
|
||||||
|
REPO_ROOT / "scripts" / "notebook_validator.py",
|
||||||
|
)
|
||||||
|
assert spec and spec.loader
|
||||||
|
mod = importlib.util.module_from_spec(spec)
|
||||||
|
# dataclasses resolves annotations through sys.modules, so the module has to
|
||||||
|
# be registered under its own name before it executes.
|
||||||
|
sys.modules[spec.name] = mod
|
||||||
|
try:
|
||||||
|
spec.loader.exec_module(mod)
|
||||||
|
except Exception:
|
||||||
|
sys.modules.pop(spec.name, None)
|
||||||
|
raise
|
||||||
|
return mod
|
||||||
|
|
||||||
|
|
||||||
|
def _load_install_python_stack():
|
||||||
|
studio_dir = REPO_ROOT / "studio"
|
||||||
|
if str(studio_dir) not in sys.path:
|
||||||
|
sys.path.insert(0, str(studio_dir))
|
||||||
|
import install_python_stack
|
||||||
|
|
||||||
|
return install_python_stack
|
||||||
|
|
||||||
|
|
||||||
|
def test_torch211_rejects_torchcodec_010(monkeypatch):
|
||||||
|
"""The guard must not be silent on the torch minor where the mismatch happens."""
|
||||||
|
import importlib.metadata
|
||||||
|
|
||||||
|
fixes = _load_import_fixes_module()
|
||||||
|
_stub_torch(monkeypatch, "2.11.0+cu128")
|
||||||
|
monkeypatch.setattr(
|
||||||
|
importlib.metadata,
|
||||||
|
"version",
|
||||||
|
lambda _name: "0.10.0+cu128",
|
||||||
|
)
|
||||||
|
|
||||||
|
hint = fixes._torchcodec_version_mismatch_hint()
|
||||||
|
assert hint is not None, "torch 2.11 + torchcodec 0.10 must not go unreported"
|
||||||
|
assert "torchcodec 0.10.0+cu128" in hint
|
||||||
|
assert "audio-torch211" in hint
|
||||||
|
assert ">=0.11" in hint
|
||||||
|
assert "<0.12.0" in hint
|
||||||
|
assert "audio-torch210" not in hint
|
||||||
|
|
||||||
|
|
||||||
|
def test_torch211_accepts_torchcodec_011(monkeypatch):
|
||||||
|
import importlib.metadata
|
||||||
|
|
||||||
|
fixes = _load_import_fixes_module()
|
||||||
|
_stub_torch(monkeypatch, "2.11.0+cu128")
|
||||||
|
monkeypatch.setattr(
|
||||||
|
importlib.metadata,
|
||||||
|
"version",
|
||||||
|
lambda _name: "0.11.1+cu128",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert fixes._torchcodec_version_mismatch_hint() is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_torch211_accepts_abi_stable_torchcodec(monkeypatch):
|
||||||
|
"""torchcodec 0.12+ targets torch >=2.11, so it is not locked to one minor."""
|
||||||
|
import importlib.metadata
|
||||||
|
|
||||||
|
fixes = _load_import_fixes_module()
|
||||||
|
for torch_version in ("2.11.0+cu128", "2.12.0", "2.13.0+cu130"):
|
||||||
|
for codec_version in ("0.12.0", "0.15.0+cu130"):
|
||||||
|
_stub_torch(monkeypatch, torch_version)
|
||||||
|
monkeypatch.setattr(importlib.metadata, "version", lambda _name, _v = codec_version: _v)
|
||||||
|
assert (
|
||||||
|
fixes._torchcodec_version_mismatch_hint() is None
|
||||||
|
), f"{torch_version} + torchcodec {codec_version} is supported upstream"
|
||||||
|
|
||||||
|
|
||||||
|
def test_torch210_still_rejects_abi_stable_torchcodec(monkeypatch):
|
||||||
|
"""The ABI-stable floor starts at torch 2.11: 2.10 keeps the exact pairing."""
|
||||||
|
import importlib.metadata
|
||||||
|
|
||||||
|
fixes = _load_import_fixes_module()
|
||||||
|
_stub_torch(monkeypatch, "2.10.0+cu128")
|
||||||
|
monkeypatch.setattr(importlib.metadata, "version", lambda _name: "0.15.0")
|
||||||
|
|
||||||
|
hint = fixes._torchcodec_version_mismatch_hint()
|
||||||
|
assert hint is not None
|
||||||
|
assert "audio-torch210" in hint
|
||||||
|
|
||||||
|
|
||||||
|
def test_torch_past_last_lockstep_row_rejects_legacy_torchcodec(monkeypatch):
|
||||||
|
"""torch newer than the last lockstep row only pairs with the 0.12+ line.
|
||||||
|
|
||||||
|
torchcodec 0.11 is pinned to torch 2.11 exactly (upstream compatibility
|
||||||
|
table), so torch 2.12/2.13 with a pre-0.12 codec must still be reported even
|
||||||
|
though the matrix has no row for those torch minors.
|
||||||
|
"""
|
||||||
|
import importlib.metadata
|
||||||
|
|
||||||
|
fixes = _load_import_fixes_module()
|
||||||
|
for torch_version in ("2.12.1+cu130", "2.13.0"):
|
||||||
|
for codec_version in ("0.11.1", "0.10.0"):
|
||||||
|
_stub_torch(monkeypatch, torch_version)
|
||||||
|
monkeypatch.setattr(importlib.metadata, "version", lambda _name, _v = codec_version: _v)
|
||||||
|
hint = fixes._torchcodec_version_mismatch_hint()
|
||||||
|
assert hint is not None, f"{torch_version} + torchcodec {codec_version} must warn"
|
||||||
|
assert "torchcodec>=0.12.0" in hint
|
||||||
|
# No audio-torch2xx extra exists for these minors, so none is offered.
|
||||||
|
assert "unsloth[audio-torch" not in hint
|
||||||
|
|
||||||
|
|
||||||
|
def test_torch_below_the_table_stays_silent(monkeypatch):
|
||||||
|
"""A torch minor older than the matrix keeps the original no-opinion behaviour."""
|
||||||
|
import importlib.metadata
|
||||||
|
|
||||||
|
fixes = _load_import_fixes_module()
|
||||||
|
_stub_torch(monkeypatch, "2.4.0")
|
||||||
|
monkeypatch.setattr(importlib.metadata, "version", lambda _name: "0.0.3")
|
||||||
|
assert fixes._torchcodec_version_mismatch_hint() is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_notebook_validator_rejects_legacy_codec_past_last_lockstep_row():
|
||||||
|
"""The mirrored notebook rule must flag the same pairing as import_fixes."""
|
||||||
|
nv = _load_notebook_validator_module()
|
||||||
|
|
||||||
|
cell = '!pip install --no-deps "torch==2.12.1" "torchcodec==0.11.1"'
|
||||||
|
findings = nv.rule_inst_004_torchcodec_torch(cell, {}, "nb.ipynb", 0)
|
||||||
|
assert len(findings) == 1
|
||||||
|
assert findings[0].rule == "R-INST-004"
|
||||||
|
assert findings[0].severity == "error"
|
||||||
|
assert "torchcodec>=0.12.0" in findings[0].hint
|
||||||
|
|
||||||
|
# Torch older than the table is still out of scope.
|
||||||
|
old = '!pip install --no-deps "torch==2.4.0" "torchcodec==0.0.3"'
|
||||||
|
assert nv.rule_inst_004_torchcodec_torch(old, {}, "nb.ipynb", 0) == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_notebook_validator_allows_abi_stable_pairing():
|
||||||
|
"""R-INST-004 is an error-severity rule: it must not fire on torch 2.11 + 0.12+."""
|
||||||
|
nv = _load_notebook_validator_module()
|
||||||
|
|
||||||
|
cell = '!pip install --no-deps "torch==2.11.0" "torchcodec==0.15.0"'
|
||||||
|
assert nv.rule_inst_004_torchcodec_torch(cell, {}, "nb.ipynb", 0) == []
|
||||||
|
|
||||||
|
stale = '!pip install --no-deps "torch==2.11.0" "torchcodec==0.10.0"'
|
||||||
|
findings = nv.rule_inst_004_torchcodec_torch(stale, {}, "nb.ipynb", 0)
|
||||||
|
assert len(findings) == 1
|
||||||
|
assert findings[0].rule == "R-INST-004"
|
||||||
|
|
||||||
|
|
||||||
|
def test_pyproject_declares_torch211_audio_extra_with_python_gate():
|
||||||
|
text = PYPROJECT.read_text(encoding = "utf-8")
|
||||||
|
match = re.search(r"^audio-torch211 = \[(.*?)^\]", text, re.MULTILINE | re.DOTALL)
|
||||||
|
assert match is not None, "pyproject must declare an audio-torch211 extra"
|
||||||
|
assert "torchcodec>=0.11.0,<0.12.0" in match.group(1)
|
||||||
|
assert "python_version >= '3.10'" in match.group(1)
|
||||||
|
|
||||||
|
|
||||||
|
def test_extras_no_deps_has_no_unconditional_torchcodec_pin():
|
||||||
|
"""A flat pin cannot serve both torch lines, so the installer picks the spec."""
|
||||||
|
lines = [
|
||||||
|
line.strip()
|
||||||
|
for line in EXTRAS_NO_DEPS_TXT.read_text(encoding = "utf-8").splitlines()
|
||||||
|
if line.strip() and not line.strip().startswith("#")
|
||||||
|
]
|
||||||
|
assert not any(line.lower().startswith("torchcodec") for line in lines), (
|
||||||
|
"extras-no-deps.txt must not pin torchcodec unconditionally; "
|
||||||
|
"install_python_stack._select_torchcodec_spec picks it per torch minor"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_select_torchcodec_spec_tracks_torch_minor():
|
||||||
|
ips = _load_install_python_stack()
|
||||||
|
assert ips._select_torchcodec_spec("2.11.0+cu128") == "torchcodec>=0.11.0,<0.12.0"
|
||||||
|
assert ips._select_torchcodec_spec("2.10.0+cu130") == "torchcodec>=0.10.0,<0.11.0"
|
||||||
|
assert ips._select_torchcodec_spec("2.9.1+cu128") == "torchcodec>=0.8.0,<0.10.0"
|
||||||
|
assert ips._select_torchcodec_spec("2.8.0+cu126") == "torchcodec>=0.6.0,<0.8.0"
|
||||||
|
|
||||||
|
|
||||||
|
def test_select_torchcodec_spec_never_caps_newer_torch_to_the_011_line():
|
||||||
|
"""0.11 is locked to torch 2.11 exactly and is absent from newer CUDA indexes,
|
||||||
|
so torch >2.11 must land on the open ABI-stable floor, not on <0.12.0."""
|
||||||
|
ips = _load_install_python_stack()
|
||||||
|
for version in ("2.12.0", "2.12.1+cu132", "2.13.0+cu130", "2.99.0"):
|
||||||
|
spec = ips._select_torchcodec_spec(version)
|
||||||
|
assert spec == ips._TORCHCODEC_ABI_STABLE_SPEC, version
|
||||||
|
assert "<" not in spec, version
|
||||||
|
|
||||||
|
|
||||||
|
def test_select_torchcodec_spec_falls_back_on_unknown_torch():
|
||||||
|
ips = _load_install_python_stack()
|
||||||
|
for value in (None, "", "not-a-version", "3.0.0", "2.rc1"):
|
||||||
|
assert ips._select_torchcodec_spec(value) == ips._TORCHCODEC_DEFAULT_SPEC
|
||||||
|
|
||||||
|
|
||||||
|
def test_select_torchcodec_spec_matches_pyproject_audio_extras():
|
||||||
|
"""The installer's specs and the pip extras must not drift apart."""
|
||||||
|
ips = _load_install_python_stack()
|
||||||
|
text = PYPROJECT.read_text(encoding = "utf-8")
|
||||||
|
for torch_version, extra in (
|
||||||
|
("2.11.0", "audio-torch211"),
|
||||||
|
("2.10.0", "audio-torch210"),
|
||||||
|
("2.9.0", "audio-torch290"),
|
||||||
|
("2.8.0", "audio-torch280"),
|
||||||
|
):
|
||||||
|
match = re.search(rf"^{extra} = \[(.*?)^\]", text, re.MULTILINE | re.DOTALL)
|
||||||
|
assert match is not None, extra
|
||||||
|
assert ips._select_torchcodec_spec(torch_version) in match.group(1), extra
|
||||||
|
|
||||||
|
|
||||||
|
def test_select_torchcodec_spec_matches_compat_matrix():
|
||||||
|
"""Installer specs must admit exactly the minors the compat matrix allows."""
|
||||||
|
from packaging.specifiers import SpecifierSet
|
||||||
|
|
||||||
|
fixes = _load_import_fixes_module()
|
||||||
|
ips = _load_install_python_stack()
|
||||||
|
probes = [f"0.{n}.0" for n in range(0, 16)]
|
||||||
|
for torch_minor, allowed in fixes._TORCH_TORCHCODEC_MINORS.items():
|
||||||
|
specifier = SpecifierSet(
|
||||||
|
ips._select_torchcodec_spec(f"{torch_minor}.0").split("torchcodec", 1)[1]
|
||||||
|
)
|
||||||
|
admitted = {p.rsplit(".", 1)[0] for p in probes if specifier.contains(p)}
|
||||||
|
assert admitted == allowed, (
|
||||||
|
f"torch {torch_minor}: installer admits {sorted(admitted)}, "
|
||||||
|
f"matrix allows {sorted(allowed)}"
|
||||||
|
)
|
||||||
|
|
|
||||||
|
|
@ -1534,7 +1534,14 @@ def patch_torchcodec_audio_decoder():
|
||||||
|
|
||||||
|
|
||||||
# torch.minor -> compatible torchcodec.minor strings (see notebook_validator.py).
|
# torch.minor -> compatible torchcodec.minor strings (see notebook_validator.py).
|
||||||
|
# torchcodec ships no `Requires-Dist: torch`, so pip cannot catch a mismatch:
|
||||||
|
# this table is the only check. It covers the lockstep releases (up to 0.11, each
|
||||||
|
# built against one torch minor); torchcodec >= 0.12 is handled by the ABI-stable
|
||||||
|
# rule below. The 2.11 row is 0.11 because that is the release upstream pairs
|
||||||
|
# with torch 2.11 exactly, and the only one on the cu128 index install.sh
|
||||||
|
# resolves torch 2.11.0 from (0.12 dropped CUDA 12.8).
|
||||||
_TORCH_TORCHCODEC_MINORS: dict[str, set[str]] = {
|
_TORCH_TORCHCODEC_MINORS: dict[str, set[str]] = {
|
||||||
|
"2.11": {"0.11"},
|
||||||
"2.10": {"0.10"},
|
"2.10": {"0.10"},
|
||||||
"2.9": {"0.8", "0.9"},
|
"2.9": {"0.8", "0.9"},
|
||||||
"2.8": {"0.6", "0.7"},
|
"2.8": {"0.6", "0.7"},
|
||||||
|
|
@ -1544,6 +1551,21 @@ _TORCH_TORCHCODEC_MINORS: dict[str, set[str]] = {
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# torch.minor -> the pyproject extra that pins the matching torchcodec line.
|
||||||
|
_TORCH_TORCHCODEC_EXTRAS: dict[str, str] = {
|
||||||
|
"2.11": "audio-torch211",
|
||||||
|
"2.10": "audio-torch210",
|
||||||
|
}
|
||||||
|
|
||||||
|
# torchcodec 0.12 onwards is ABI-stable against torch >= 2.11 (its build sets
|
||||||
|
# TORCH_TARGET_VERSION to 2.11), so that half of the matrix is open-ended and
|
||||||
|
# cannot be written as a finite set of minors: any torchcodec >= 0.12 pairs with
|
||||||
|
# any torch >= 2.11. Everything older is locked to a single torch minor, which is
|
||||||
|
# what the table above encodes.
|
||||||
|
_TORCHCODEC_ABI_STABLE_TORCH = (2, 11)
|
||||||
|
_TORCHCODEC_ABI_STABLE_CODEC = (0, 12)
|
||||||
|
|
||||||
|
|
||||||
def _torchcodec_exclusive_upper(pin: str) -> str:
|
def _torchcodec_exclusive_upper(pin: str) -> str:
|
||||||
"""Next torchcodec minor as an exclusive pip upper bound (0.10 -> <0.11.0)."""
|
"""Next torchcodec minor as an exclusive pip upper bound (0.10 -> <0.11.0)."""
|
||||||
major, minor = pin.split(".", 1)
|
major, minor = pin.split(".", 1)
|
||||||
|
|
@ -1561,25 +1583,43 @@ def _torchcodec_version_mismatch_hint() -> str | None:
|
||||||
except Exception:
|
except Exception:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def _minor(version: str) -> str:
|
def _release(version: str) -> tuple:
|
||||||
parts = Version(version.split("+", 1)[0]).release
|
parts = Version(version.split("+", 1)[0]).release
|
||||||
return ".".join(str(p) for p in parts[:2])
|
return tuple(parts[:2]) + (0,) * (2 - len(parts[:2]))
|
||||||
|
|
||||||
try:
|
try:
|
||||||
torch_minor = _minor(torch.__version__)
|
torch_release = _release(torch.__version__)
|
||||||
codec_minor = _minor(torchcodec_version)
|
codec_release = _release(torchcodec_version)
|
||||||
except Exception:
|
except Exception:
|
||||||
# Non-PEP440 version strings must never break `import unsloth`.
|
# Non-PEP440 version strings must never break `import unsloth`.
|
||||||
return None
|
return None
|
||||||
|
if (
|
||||||
|
torch_release >= _TORCHCODEC_ABI_STABLE_TORCH
|
||||||
|
and codec_release >= _TORCHCODEC_ABI_STABLE_CODEC
|
||||||
|
):
|
||||||
|
return None # ABI-stable pairing, not locked to one torch minor
|
||||||
|
torch_minor = ".".join(str(p) for p in torch_release)
|
||||||
|
codec_minor = ".".join(str(p) for p in codec_release)
|
||||||
allowed = _TORCH_TORCHCODEC_MINORS.get(torch_minor)
|
allowed = _TORCH_TORCHCODEC_MINORS.get(torch_minor)
|
||||||
if allowed is None or codec_minor in allowed:
|
if allowed is None:
|
||||||
|
# No lockstep row for this torch minor. Torch older than the table is
|
||||||
|
# out of scope, so stay silent. Torch at or past the ABI-stable floor
|
||||||
|
# only pairs with the open-ended line (>= 0.12), and the early return
|
||||||
|
# above already cleared those, so whatever reaches here is a legacy
|
||||||
|
# single-minor codec built for an older torch: point at the 0.12+ line.
|
||||||
|
if torch_release < _TORCHCODEC_ABI_STABLE_TORCH:
|
||||||
|
return None
|
||||||
|
abi_pin = ".".join(str(p) for p in _TORCHCODEC_ABI_STABLE_CODEC)
|
||||||
|
install_hint = f"`pip install 'torchcodec>={abi_pin}.0'`"
|
||||||
|
elif codec_minor in allowed:
|
||||||
return None
|
return None
|
||||||
|
else:
|
||||||
pin = sorted(allowed)[-1]
|
pin = sorted(allowed)[-1]
|
||||||
upper = _torchcodec_exclusive_upper(pin)
|
upper = _torchcodec_exclusive_upper(pin)
|
||||||
install_hint = f"`pip install 'torchcodec>={pin},{upper}'`"
|
install_hint = f"`pip install 'torchcodec>={pin},{upper}'`"
|
||||||
if torch_minor == "2.10":
|
extra = _TORCH_TORCHCODEC_EXTRAS.get(torch_minor)
|
||||||
install_hint += " or `pip install 'unsloth[audio-torch210]'`"
|
if extra is not None:
|
||||||
|
install_hint += f" or `pip install 'unsloth[{extra}]'`"
|
||||||
return (
|
return (
|
||||||
f"torchcodec {torchcodec_version} is incompatible with torch {torch.__version__}; "
|
f"torchcodec {torchcodec_version} is incompatible with torch {torch.__version__}; "
|
||||||
f"install a matching build with {install_hint}."
|
f"install a matching build with {install_hint}."
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue