diff --git a/install.sh b/install.sh index 0370559540..81c50bc899 100755 --- a/install.sh +++ b/install.sh @@ -1442,8 +1442,14 @@ if [ "$_NO_TORCH_FLAG" = true ] || [ "$MAC_INTEL" = true ]; then SKIP_TORCH=true fi +# Apple Silicon: exclude broken mlx-lm 0.31.3 (QK-norm load regression for +# gemma4 / qwen3_5; mlx-lm #1242). A curl-piped install has no overrides file +# and skips the guarded MLX step (SKIP_STUDIO_BASE=1), so this is the only cover. +_MLX_LM_EXCLUDE_ARG="" + # Apple Silicon: override mlx-vlm / mlx-lm's transformers pin (see overrides file). if [ "$OS" = "macos" ] && [ "$_ARCH" = "arm64" ]; then + _MLX_LM_EXCLUDE_ARG="mlx-lm!=0.31.3" _OVERRIDES_FILE="$(cd "$(dirname "$0" 2>/dev/null || echo ".")" && pwd)/studio/backend/requirements/single-env/overrides-darwin-arm64.txt" if [ -f "$_OVERRIDES_FILE" ]; then # uv splits UV_OVERRIDE on whitespace, so a repo path with whitespace @@ -2679,9 +2685,11 @@ if [ "$_MIGRATED" = true ]; then run_install_cmd_retry "install no-torch runtime deps" uv pip install --python "$_VENV_PY" --no-deps -r "$_NO_TORCH_RT" fi else + # Pin mlx-lm away from 0.31.3 here too: a curl-piped migration has no + # overrides file, so UV_OVERRIDE is unset and this positional is the only cover. run_install_cmd_retry "install unsloth (migrated)" uv pip install --python "$_VENV_PY" \ --reinstall-package unsloth --reinstall-package unsloth-zoo \ - "unsloth>=2026.6.9" "unsloth-zoo>=2026.6.7" + "unsloth>=2026.6.9" "unsloth-zoo>=2026.6.7" ${_MLX_LM_EXCLUDE_ARG:-} fi if [ "$STUDIO_LOCAL_INSTALL" = true ]; then substep "overlaying local repo (editable)..." @@ -2912,7 +2920,7 @@ elif [ -n "$TORCH_INDEX_URL" ]; then "unsloth-zoo @ git+https://github.com/unslothai/unsloth-zoo" else run_install_cmd_retry "install unsloth" uv pip install --python "$_VENV_PY" \ - --upgrade-package unsloth -- "$PACKAGE_NAME" + --upgrade-package unsloth -- "$PACKAGE_NAME" ${_MLX_LM_EXCLUDE_ARG:-} fi # AMD ROCm: repair torch if the unsloth/unsloth-zoo install pulled in # CUDA torch from PyPI, overwriting the ROCm wheels installed in Step 1. diff --git a/scripts/scan_packages_baseline.json b/scripts/scan_packages_baseline.json index d42225e205..1d34cfb66d 100644 --- a/scripts/scan_packages_baseline.json +++ b/scripts/scan_packages_baseline.json @@ -1545,6 +1545,22 @@ "severity": "HIGH", "evidence": "Obfusc: L52: compiled = compile(source=pysrc, filename=filename, mode='exec')\nExec: L53: exec(compiled, globs, globs)", "evidence_hash": "c429e4c977a61db6b7c717b5a552fce74eda622213e49eb5467a3782fd746fb9" + }, + { + "package": "multiprocess", + "file": "multiprocess/forkserver.py", + "check": "Reverse shell / bind shell pattern", + "severity": "CRITICAL", + "evidence": "L6: import socket sha256:6c707119169286c9a798e2c8d13a48614e481d8a503950916fd4ffb4c94d3182", + "evidence_hash": "50fec0f0522a8e4e636bf348b752002d7935d8455af31fb78c6f11e2eba19f6d" + }, + { + "package": "multiprocess", + "file": "multiprocess/tests/__init__.py", + "check": "Reverse shell / bind shell pattern", + "severity": "CRITICAL", + "evidence": "L3569: os.dup2(conn.fileno(), i) | L3601: \"test needs os.dup2()\") | L3619: os.dup2(fd, newfd) | L20: import socket sha256:c824dc0f409f242420c3fbb324790c53cb3078d2c8b07ee8f2a05694b01c2946", + "evidence_hash": "3878a2b430c175dbc5877a95195bfe52f9588ff73fb74e2261ed5e33087915ad" } ] } diff --git a/studio/backend/requirements/single-env/overrides-darwin-arm64.txt b/studio/backend/requirements/single-env/overrides-darwin-arm64.txt index 56b948644f..43f37b3183 100644 --- a/studio/backend/requirements/single-env/overrides-darwin-arm64.txt +++ b/studio/backend/requirements/single-env/overrides-darwin-arm64.txt @@ -10,3 +10,9 @@ transformers>=4.57.6 # anyio that also ImportErrors on TaskHandle and 500s the server. An override # wins the fight, so force one consistent <4.14 here too. anyio<4.14.0 + +# mlx-lm 0.31.3 regressed QK-norm archs (gemma4 / qwen3_5): strict load_weights +# rejects q_norm/k_norm, so those checkpoints fail to load. mlx-lm #1242. +# The override also drops it from transitive resolution; keep the >=0.22.0 floor +# (mirrors mlx_repair.py _MLX_MIN_VERSIONS) or the resolver could go below it. +mlx-lm>=0.22.0,!=0.31.3 diff --git a/studio/backend/tests/test_mlx_repair.py b/studio/backend/tests/test_mlx_repair.py index 1b0cbf9df1..365cc46410 100644 --- a/studio/backend/tests/test_mlx_repair.py +++ b/studio/backend/tests/test_mlx_repair.py @@ -271,6 +271,29 @@ def test_stack_available_requires_runtime_imports_and_versions(monkeypatch): assert imported == list(mr._MLX_RUNTIME_IMPORTS) +def test_mlx_packages_exclude_known_bad_mlx_lm(): + # mlx-lm 0.31.3 regressed QK-norm archs (gemma4 / qwen3_5); the install spec + # must exclude it so the resolver picks 0.31.2 or >=0.31.4. See mlx-lm #1242. + (mlx_lm_spec,) = [p for p in mr.MLX_PACKAGES if p.startswith("mlx-lm")] + assert mlx_lm_spec == "mlx-lm>=0.22.0,!=0.31.3" + + +@pytest.mark.parametrize("bad_form", ["0.31.3", "0.31.3.0"]) +def test_known_bad_installed_mlx_lm_triggers_repair(monkeypatch, bad_form): + # An installed 0.31.3 counts as unsatisfied so the self-heal replaces it; + # parsed-Version compare also catches the trailing-zero form 0.31.3.0. + import importlib.metadata as metadata + + def _version(name): + return bad_form if name == "mlx-lm" else mr._MLX_MIN_VERSIONS[name] + + monkeypatch.setattr(metadata, "version", _version) + monkeypatch.setattr( + mr.importlib, "import_module", lambda _n: pytest.fail("versions must gate imports") + ) + assert mr.mlx_stack_available() is False + + def test_no_op_off_apple_silicon(monkeypatch): monkeypatch.setattr(mr, "is_apple_silicon", lambda: False) called = {"n": 0} diff --git a/studio/backend/utils/mlx_repair.py b/studio/backend/utils/mlx_repair.py index 520c11c3b1..7e1c9864c9 100644 --- a/studio/backend/utils/mlx_repair.py +++ b/studio/backend/utils/mlx_repair.py @@ -45,9 +45,21 @@ DISABLE_ENV_VAR = "UNSLOTH_DISABLE_MLX_AUTOREPAIR" # deps). mlx-vlm especially must be >=0.4.4: an older one still imports but # breaks VLM Train/Export, so installing it would wrongly clear chat-only. _MLX_MIN_VERSIONS = {"mlx": "0.22.0", "mlx-lm": "0.22.0", "mlx-vlm": "0.4.4"} +# mlx-lm 0.31.3 regressed QK-norm archs (gemma4 / qwen3_5): strict load_weights +# rejects q_norm/k_norm, so a self-heal must not pull it. mlx-lm #1242. +_MLX_BAD_VERSIONS = {"mlx-lm": ("0.31.3",)} _MLX_PACKAGE_NAMES = tuple(_MLX_MIN_VERSIONS) _MLX_RUNTIME_IMPORTS = ("mlx.core", "mlx_lm", "mlx_lm.sample_utils", "mlx_vlm") -MLX_PACKAGES = tuple(f"{name}>={version}" for name, version in _MLX_MIN_VERSIONS.items()) + + +def _mlx_spec(name: str, version: str) -> str: + spec = f"{name}>={version}" + for bad in _MLX_BAD_VERSIONS.get(name, ()): + spec += f",!={bad}" + return spec + + +MLX_PACKAGES = tuple(_mlx_spec(name, version) for name, version in _MLX_MIN_VERSIONS.items()) _MLX_REINSTALL_ARGS = tuple( arg for name in _MLX_PACKAGE_NAMES for arg in ("--reinstall-package", name) ) @@ -140,7 +152,12 @@ def _mlx_versions_satisfy_minimums() -> bool: return False for name, minimum in _MLX_MIN_VERSIONS.items(): try: - if Version(_dist_version(name)) < Version(minimum): + installed = Version(_dist_version(name)) + if installed < Version(minimum): + return False + # A known-broken build counts as unsatisfied so the self-heal + # reinstalls a good one; Version compare matches 0.31.3(.0/+local). + if any(installed == Version(bad) for bad in _MLX_BAD_VERSIONS.get(name, ())): return False except PackageNotFoundError: return False diff --git a/studio/install_python_stack.py b/studio/install_python_stack.py index 439d3ffe7b..e033a56a0a 100644 --- a/studio/install_python_stack.py +++ b/studio/install_python_stack.py @@ -1532,6 +1532,10 @@ LOCAL_DD_UNSTRUCTURED_PLUGIN = ( ) LOCAL_DD_GITHUB_PLUGIN = SCRIPT_DIR / "backend" / "plugins" / "data-designer-github-repo-seed" +# mlx-lm 0.31.3 broke gemma4 / qwen3_5 loading (strict load_weights rejects the +# QK-norm q_norm/k_norm tensors); exclude just that release. See mlx-lm #1242. +MLX_LM_BAD_VERSION_EXCLUSION = "!=0.31.3" + # Apple Silicon: override mlx-vlm/mlx-lm's transformers pin (see overrides). # _uv_safe_path: uv truncates UV_OVERRIDE at the first space too (issue #6503). _MLX_OVERRIDES = SINGLE_ENV / "overrides-darwin-arm64.txt" @@ -2092,6 +2096,8 @@ def install_python_stack() -> int: # macOS arm64: install MLX stack at latest (UV_OVERRIDE relaxes the # mlx-vlm / mlx-lm transformers pin -- set at module load). + # Exclude mlx-lm 0.31.3 (see MLX_LM_BAD_VERSION_EXCLUSION); it broke + # gemma4 / qwen3_5 QK-norm loading. mlx-lm #1242. if IS_MAC_ARM and not skip_base: _progress("MLX stack (Apple Silicon)") pip_install( @@ -2100,7 +2106,7 @@ def install_python_stack() -> int: "--upgrade", "mlx", "mlx-metal", - "mlx-lm", + f"mlx-lm{MLX_LM_BAD_VERSION_EXCLUSION}", "mlx-vlm", )