diff --git a/scripts/scan_packages_baseline.json b/scripts/scan_packages_baseline.json index 67952c24f1..f953f4d206 100644 --- a/scripts/scan_packages_baseline.json +++ b/scripts/scan_packages_baseline.json @@ -72,13 +72,6 @@ "severity": "CRITICAL", "evidence": "Archive: L317: a['TarFileType'] = tarfile.open(fileobj=_fileW,mode='w')\nNetwork: L330: x['SocketType'] = _socket = socket.socket()" }, - { - "package": "evaluate", - "file": "evaluate/utils/file_utils.py", - "check": "C2 polling/beaconing loop detected", - "severity": "CRITICAL", - "evidence": "L261: while True:" - }, { "package": "execnet", "file": "execnet/gateway_base.py", @@ -401,27 +394,6 @@ "severity": "CRITICAL", "evidence": "Base64: L488: decoded_bytes = base64.b64decode(base64_encoded)\nSubprocess: L80: return subprocess.call(['which', name], | L100: p = subprocess.Popen(['pbcopy', 'w'], | L105: p = subprocess.Popen(['pbpaste', 'r']," }, - { - "package": "pytest", - "file": "_pytest/_py/path.py", - "check": "Downloads and executes remote code", - "severity": "CRITICAL", - "evidence": "L1153: exec(f.read(), mod.__dict__)" - }, - { - "package": "pytest", - "file": "_pytest/capture.py", - "check": "Reverse shell / bind shell pattern", - "severity": "CRITICAL", - "evidence": "L483: os.dup2(self.targetfd_invalid, targetfd) | L522: os.dup2(self.tmpfile.fileno(), self.targetfd) | L532: os.dup2(self.targetfd_save, self.targetfd)" - }, - { - "package": "pytest", - "file": "_pytest/config/__init__.py", - "check": "Reverse shell / bind shell pattern", - "severity": "CRITICAL", - "evidence": "L260: os.dup2(devnull, sys.stdout.fileno())" - }, { "package": "python-dateutil", "file": "dateutil/__init__.py", @@ -527,6 +499,34 @@ "severity": "CRITICAL", "evidence": "L13: __import__(__package__ + '.linalg') | L14: __import__(__package__ + '.fft')" }, + { + "package": "scipy", + "file": "scipy/_external/array_api_compat/cupy/__init__.py", + "check": "Downloads and executes remote code", + "severity": "CRITICAL", + "evidence": "L12: __import__(__package__ + '.linalg') | L13: __import__(__package__ + '.fft')" + }, + { + "package": "scipy", + "file": "scipy/_external/array_api_compat/dask/array/__init__.py", + "check": "Downloads and executes remote code", + "severity": "CRITICAL", + "evidence": "L16: __import__(__package__ + '.linalg') | L17: __import__(__package__ + '.fft')" + }, + { + "package": "scipy", + "file": "scipy/_external/array_api_compat/numpy/__init__.py", + "check": "Downloads and executes remote code", + "severity": "CRITICAL", + "evidence": "L23: __import__(__package__ + \".linalg\") | L25: __import__(__package__ + \".fft\")" + }, + { + "package": "scipy", + "file": "scipy/_external/array_api_compat/torch/__init__.py", + "check": "Downloads and executes remote code", + "severity": "CRITICAL", + "evidence": "L13: __import__(__package__ + '.linalg') | L14: __import__(__package__ + '.fft')" + }, { "package": "sentencepiece", "file": "sentencepiece/__init__.py", @@ -814,6 +814,13 @@ "severity": "CRITICAL", "evidence": "L67: input_gguf=\"/tmp/in.gguf\"," }, + { + "package": "unsloth-zoo", + "file": "tests/test_mlx_save_export_regressions.py", + "check": "Writes to /tmp and executes (staged dropper)", + "severity": "CRITICAL", + "evidence": "L164: temporary_location=\"/tmp/ignored\"," + }, { "package": "unsloth-zoo", "file": "tests/test_upstream_pinned_symbols_transformers.py", @@ -933,13 +940,6 @@ "severity": "HIGH", "evidence": "Key: L187: \"-----BEGIN PUBLIC KEY-----\", | L188: \"-----BEGIN RSA PUBLIC KEY-----\",\nNetwork: L225: http_client: httpx.AsyncClient | None = None, | L411: else httpx.AsyncClient(timeout=httpx.Timeout(10.0))" }, - { - "package": "hypothesis", - "file": "hypothesis/internal/scrutineer.py", - "check": "Anti-analysis/sandbox evasion + suspicious behavior", - "severity": "HIGH", - "evidence": "Anti: L76: return sys.gettrace() is None | L113: sys.settrace(self.trace) | L136: sys.settrace(None)" - }, { "package": "ipython", "file": "IPython/core/debugger.py", @@ -982,20 +982,6 @@ "severity": "HIGH", "evidence": "Obfusc: L709: return compile(source, filename, \"exec\")\nExec: L1228: exec(code, namespace)" }, - { - "package": "kgb", - "file": "kgb/spies.py", - "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", - "severity": "HIGH", - "evidence": "Obfusc: L934: eval(compile(func_code_str, '', 'exec'),\nExec: L934: eval(compile(func_code_str, '', 'exec')," - }, - { - "package": "langid", - "file": "langid/train/common.py", - "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", - "severity": "HIGH", - "evidence": "Obfusc: L44: yield marshal.load(t)\nExec: L85: key = eval(row[0])" - }, { "package": "matplotlib", "file": "matplotlib/sphinxext/plot_directive.py", @@ -1108,20 +1094,6 @@ "severity": "HIGH", "evidence": "Obfusc: L45: mod = __import__(module_name, None, None, ['__all__'])\nExec: L154: exec(f.read(), custom_namespace)" }, - { - "package": "pytest", - "file": "_pytest/_py/path.py", - "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", - "severity": "HIGH", - "evidence": "Obfusc: L626: mod = __import__(hashtype) | L1118: __import__(modname)\nExec: L1153: exec(f.read(), mod.__dict__)" - }, - { - "package": "pytest", - "file": "_pytest/assertion/rewrite.py", - "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", - "severity": "HIGH", - "evidence": "Obfusc: L393: co = marshal.load(fp) | L395: trace(f\"_read_pyc({source}): marshal.load error {e}\")\nExec: L188: exec(co, module.__dict__)" - }, { "package": "scikit-learn", "file": "sklearn/externals/array_api_compat/torch/__init__.py", @@ -1318,6 +1290,13 @@ "severity": "HIGH", "evidence": "Obfusc: L3078: module = __import__('transformers', fromlist=[model_class_name])\nExec: L2960: exec(f\"from transformers.modeling_utils import ({', '.join(functions)})\", locals(), globals()) | L3006: exec(save_pretrained, globals(), functions)" }, + { + "package": "unsloth-zoo", + "file": "tests/test_mlx_trainer_internals.py", + "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", + "severity": "HIGH", + "evidence": "Obfusc: L430: assert ppl == pytest.approx(__import__(\"math\").exp(2.5))\nExec: L408: def eval(self):" + }, { "package": "werkzeug", "file": "werkzeug/routing/rules.py", diff --git a/studio/backend/requirements/extras-no-deps.txt b/studio/backend/requirements/extras-no-deps.txt index 23c61baa44..5830a47789 100644 --- a/studio/backend/requirements/extras-no-deps.txt +++ b/studio/backend/requirements/extras-no-deps.txt @@ -11,10 +11,12 @@ peft==0.18.1 # TRL and related packages trl==0.23.1 -git+https://github.com/meta-pytorch/OpenEnv.git # executorch>=1.0.1 # 41.5 MB - no imports in unsloth/zoo/studio torch-c-dlpack-ext sentence_transformers==5.2.0 transformers==4.57.6 pytorch_tokenizers kernels==0.12.1 +# kernels<3.11 imports tomli as its tomllib fallback; --no-deps skips its own +# marker dep, so list it here (no-op on the 3.12/3.13 default installs). +tomli; python_version < "3.11" diff --git a/studio/backend/requirements/extras.txt b/studio/backend/requirements/extras.txt index b56c2dfa40..1baf2b6f2d 100644 --- a/studio/backend/requirements/extras.txt +++ b/studio/backend/requirements/extras.txt @@ -1,27 +1,11 @@ -# OpenEnv dependencies -tomli -tomli-w - -# ExecuTorch dependencies -ruamel.yaml -# coremltools # 10.2 MB - Apple CoreML, no imports in unsloth/zoo/studio -expecttest +# transitive dep of onnxruntime (via data-designer's pymupdf4llm) flatbuffers -hydra-core -hypothesis -kgb -parameterized -pytest>=9.0.3,<10 -pytest-json-report -pytest-rerunfailures>=16.2,<17 -pytest-xdist -# Also needed by sentence_transformers (installed with --no-deps in extras-no-deps.txt) +# Also needed by sentence_transformers (installed with --no-deps in extras-no-deps.txt); +# librosa pulls it in too, but is skipped in no-torch mode. scikit-learn==1.7.1 # Additional extras -pybind11 -langid -jiwer +jiwer # WER/CER metrics for vision OCR save-merge benchmarks omegaconf einx pyloudnorm @@ -44,12 +28,7 @@ pystoi soundfile tensorboard torch-stoi -evaluate timm -transformers-cfg -open_spiel -addict -easydict einops tabulate openai>=2.7.2 diff --git a/studio/backend/requirements/single-env/constraints.txt b/studio/backend/requirements/single-env/constraints.txt index 9e81880016..0ed2bf8b26 100644 --- a/studio/backend/requirements/single-env/constraints.txt +++ b/studio/backend/requirements/single-env/constraints.txt @@ -8,7 +8,7 @@ huggingface-hub==0.36.2 datasets==4.3.0 pyarrow==23.0.1 -# FastMCP/OpenEnv compat +# FastMCP compat fastmcp>=3.0.2 mcp>=1.24,<2 websockets>=15.0.1 diff --git a/studio/backend/requirements/studio.txt b/studio/backend/requirements/studio.txt index 9efa10faef..1b7f7a668c 100644 --- a/studio/backend/requirements/studio.txt +++ b/studio/backend/requirements/studio.txt @@ -9,8 +9,6 @@ pandas nest_asyncio datasets==4.3.0 pyjwt -easydict -addict # gradio>=4.0.0 # 148 MB - Studio uses React + FastAPI, not Gradio huggingface-hub==0.36.2 structlog>=24.1.0 diff --git a/studio/install_python_stack.py b/studio/install_python_stack.py index 491429a4ef..e6095748fe 100644 --- a/studio/install_python_stack.py +++ b/studio/install_python_stack.py @@ -1658,7 +1658,7 @@ def run( # Packages to skip on Windows (require special build steps) -WINDOWS_SKIP_PACKAGES = {"open_spiel", "triton_kernels"} +WINDOWS_SKIP_PACKAGES = {"triton_kernels"} # Packages to skip when torch is unavailable (Intel Mac GGUF-only mode). # These either *are* torch extensions or have unconditional @@ -1675,7 +1675,6 @@ NO_TORCH_SKIP_PACKAGES = { "torchcodec", "torch-c-dlpack-ext", "openai-whisper", - "transformers-cfg", "librosa", } diff --git a/tests/python/test_e2e_no_torch_sandbox.py b/tests/python/test_e2e_no_torch_sandbox.py index 23d4a2c2b3..f412de2063 100644 --- a/tests/python/test_e2e_no_torch_sandbox.py +++ b/tests/python/test_e2e_no_torch_sandbox.py @@ -866,13 +866,11 @@ class TestInstallPythonStackFiltering: result_path = ips._filter_requirements(extras, ips.NO_TORCH_SKIP_PACKAGES) filtered = Path(result_path).read_text(encoding = "utf-8").lower() + lines = [ + l.strip() for l in filtered.splitlines() if l.strip() and not l.strip().startswith("#") + ] - for pkg in ["torch-stoi", "timm", "openai-whisper", "transformers-cfg"]: - lines = [ - l.strip() - for l in filtered.splitlines() - if l.strip() and not l.strip().startswith("#") - ] + for pkg in ips.NO_TORCH_SKIP_PACKAGES: assert not any( l.startswith(pkg) for l in lines ), f"{pkg} should be removed from extras.txt" diff --git a/tests/python/test_no_torch_filtering.py b/tests/python/test_no_torch_filtering.py index 9dcda5c60c..732c1b7432 100644 --- a/tests/python/test_no_torch_filtering.py +++ b/tests/python/test_no_torch_filtering.py @@ -138,7 +138,6 @@ class TestFilterRequirements: req = self._write_req( tmp_path, """\ - open_spiel triton_kernels torch-stoi timm @@ -222,8 +221,8 @@ class TestRealRequirementsFiltering: filtered = self._non_blank_non_comment(Path(result)) original = self._non_blank_non_comment(EXTRAS_TXT) - # These must be gone. - for pkg in ["torch-stoi", "timm", "openai-whisper", "transformers-cfg"]: + # Every NO_TORCH skip package present in extras.txt must be gone. + for pkg in ips.NO_TORCH_SKIP_PACKAGES: assert not any( l.lower().startswith(pkg) for l in filtered ), f"{pkg} should be removed from extras.txt"