Studio: remove OpenEnv and other unused packages (#6585)
* Studio: drop OpenEnv and unused ExecuTorch/open_spiel install deps * Studio: drop 8 more unused install deps from extras * Studio: restore tomli<3.11 for kernels; tidy dep-cleanup comments and tests * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Studio: refresh scan-packages baseline for scipy _external + unsloth-zoo tests scipy moved its vendored array_api_compat from scipy/_lib to scipy/_external, so the four allowlisted array_api_compat __init__.py entries stopped matching and resurfaced as unsuppressed CRITICAL "Downloads and executes remote code" findings on all three pip scan-packages shards (extras, hf-stack, studio). Add the _external paths next to the existing _lib ones so both scipy layouts stay covered. Allowlist two unsloth-zoo test-file false positives now present in the hf-stack shard: tests/test_mlx_save_export_regressions.py (writes to /tmp dropper) and tests/test_mlx_trainer_internals.py (obfuscation plus exec/eval). Drop nine stale entries for packages removed from the Studio requirements and no longer in any shard closure (evaluate, pytest, hypothesis, kgb, langid), confirmed absent via with-deps resolution of all three shards. --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Daniel Han <danielhanchen@gmail.com>
This commit is contained in:
oobabooga
GitHub
parent
8aa27f6db3
commit
1cc785e5a0
8 changed files with 57 additions and 103 deletions
|
|
@ -72,13 +72,6 @@
|
|||
"severity": "CRITICAL",
|
||||
"evidence": "Archive: L317: a['TarFileType'] = tarfile.open(fileobj=_fileW,mode='w')\nNetwork: L330: x['SocketType'] = _socket = socket.socket()"
|
||||
},
|
||||
{
|
||||
"package": "evaluate",
|
||||
"file": "evaluate/utils/file_utils.py",
|
||||
"check": "C2 polling/beaconing loop detected",
|
||||
"severity": "CRITICAL",
|
||||
"evidence": "L261: while True:"
|
||||
},
|
||||
{
|
||||
"package": "execnet",
|
||||
"file": "execnet/gateway_base.py",
|
||||
|
|
@ -401,27 +394,6 @@
|
|||
"severity": "CRITICAL",
|
||||
"evidence": "Base64: L488: decoded_bytes = base64.b64decode(base64_encoded)\nSubprocess: L80: return subprocess.call(['which', name], | L100: p = subprocess.Popen(['pbcopy', 'w'], | L105: p = subprocess.Popen(['pbpaste', 'r'],"
|
||||
},
|
||||
{
|
||||
"package": "pytest",
|
||||
"file": "_pytest/_py/path.py",
|
||||
"check": "Downloads and executes remote code",
|
||||
"severity": "CRITICAL",
|
||||
"evidence": "L1153: exec(f.read(), mod.__dict__)"
|
||||
},
|
||||
{
|
||||
"package": "pytest",
|
||||
"file": "_pytest/capture.py",
|
||||
"check": "Reverse shell / bind shell pattern",
|
||||
"severity": "CRITICAL",
|
||||
"evidence": "L483: os.dup2(self.targetfd_invalid, targetfd) | L522: os.dup2(self.tmpfile.fileno(), self.targetfd) | L532: os.dup2(self.targetfd_save, self.targetfd)"
|
||||
},
|
||||
{
|
||||
"package": "pytest",
|
||||
"file": "_pytest/config/__init__.py",
|
||||
"check": "Reverse shell / bind shell pattern",
|
||||
"severity": "CRITICAL",
|
||||
"evidence": "L260: os.dup2(devnull, sys.stdout.fileno())"
|
||||
},
|
||||
{
|
||||
"package": "python-dateutil",
|
||||
"file": "dateutil/__init__.py",
|
||||
|
|
@ -527,6 +499,34 @@
|
|||
"severity": "CRITICAL",
|
||||
"evidence": "L13: __import__(__package__ + '.linalg') | L14: __import__(__package__ + '.fft')"
|
||||
},
|
||||
{
|
||||
"package": "scipy",
|
||||
"file": "scipy/_external/array_api_compat/cupy/__init__.py",
|
||||
"check": "Downloads and executes remote code",
|
||||
"severity": "CRITICAL",
|
||||
"evidence": "L12: __import__(__package__ + '.linalg') | L13: __import__(__package__ + '.fft')"
|
||||
},
|
||||
{
|
||||
"package": "scipy",
|
||||
"file": "scipy/_external/array_api_compat/dask/array/__init__.py",
|
||||
"check": "Downloads and executes remote code",
|
||||
"severity": "CRITICAL",
|
||||
"evidence": "L16: __import__(__package__ + '.linalg') | L17: __import__(__package__ + '.fft')"
|
||||
},
|
||||
{
|
||||
"package": "scipy",
|
||||
"file": "scipy/_external/array_api_compat/numpy/__init__.py",
|
||||
"check": "Downloads and executes remote code",
|
||||
"severity": "CRITICAL",
|
||||
"evidence": "L23: __import__(__package__ + \".linalg\") | L25: __import__(__package__ + \".fft\")"
|
||||
},
|
||||
{
|
||||
"package": "scipy",
|
||||
"file": "scipy/_external/array_api_compat/torch/__init__.py",
|
||||
"check": "Downloads and executes remote code",
|
||||
"severity": "CRITICAL",
|
||||
"evidence": "L13: __import__(__package__ + '.linalg') | L14: __import__(__package__ + '.fft')"
|
||||
},
|
||||
{
|
||||
"package": "sentencepiece",
|
||||
"file": "sentencepiece/__init__.py",
|
||||
|
|
@ -814,6 +814,13 @@
|
|||
"severity": "CRITICAL",
|
||||
"evidence": "L67: input_gguf=\"/tmp/in.gguf\","
|
||||
},
|
||||
{
|
||||
"package": "unsloth-zoo",
|
||||
"file": "tests/test_mlx_save_export_regressions.py",
|
||||
"check": "Writes to /tmp and executes (staged dropper)",
|
||||
"severity": "CRITICAL",
|
||||
"evidence": "L164: temporary_location=\"/tmp/ignored\","
|
||||
},
|
||||
{
|
||||
"package": "unsloth-zoo",
|
||||
"file": "tests/test_upstream_pinned_symbols_transformers.py",
|
||||
|
|
@ -933,13 +940,6 @@
|
|||
"severity": "HIGH",
|
||||
"evidence": "Key: L187: \"-----BEGIN PUBLIC KEY-----\", | L188: \"-----BEGIN RSA PUBLIC KEY-----\",\nNetwork: L225: http_client: httpx.AsyncClient | None = None, | L411: else httpx.AsyncClient(timeout=httpx.Timeout(10.0))"
|
||||
},
|
||||
{
|
||||
"package": "hypothesis",
|
||||
"file": "hypothesis/internal/scrutineer.py",
|
||||
"check": "Anti-analysis/sandbox evasion + suspicious behavior",
|
||||
"severity": "HIGH",
|
||||
"evidence": "Anti: L76: return sys.gettrace() is None | L113: sys.settrace(self.trace) | L136: sys.settrace(None)"
|
||||
},
|
||||
{
|
||||
"package": "ipython",
|
||||
"file": "IPython/core/debugger.py",
|
||||
|
|
@ -982,20 +982,6 @@
|
|||
"severity": "HIGH",
|
||||
"evidence": "Obfusc: L709: return compile(source, filename, \"exec\")\nExec: L1228: exec(code, namespace)"
|
||||
},
|
||||
{
|
||||
"package": "kgb",
|
||||
"file": "kgb/spies.py",
|
||||
"check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval",
|
||||
"severity": "HIGH",
|
||||
"evidence": "Obfusc: L934: eval(compile(func_code_str, '<string>', 'exec'),\nExec: L934: eval(compile(func_code_str, '<string>', 'exec'),"
|
||||
},
|
||||
{
|
||||
"package": "langid",
|
||||
"file": "langid/train/common.py",
|
||||
"check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval",
|
||||
"severity": "HIGH",
|
||||
"evidence": "Obfusc: L44: yield marshal.load(t)\nExec: L85: key = eval(row[0])"
|
||||
},
|
||||
{
|
||||
"package": "matplotlib",
|
||||
"file": "matplotlib/sphinxext/plot_directive.py",
|
||||
|
|
@ -1108,20 +1094,6 @@
|
|||
"severity": "HIGH",
|
||||
"evidence": "Obfusc: L45: mod = __import__(module_name, None, None, ['__all__'])\nExec: L154: exec(f.read(), custom_namespace)"
|
||||
},
|
||||
{
|
||||
"package": "pytest",
|
||||
"file": "_pytest/_py/path.py",
|
||||
"check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval",
|
||||
"severity": "HIGH",
|
||||
"evidence": "Obfusc: L626: mod = __import__(hashtype) | L1118: __import__(modname)\nExec: L1153: exec(f.read(), mod.__dict__)"
|
||||
},
|
||||
{
|
||||
"package": "pytest",
|
||||
"file": "_pytest/assertion/rewrite.py",
|
||||
"check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval",
|
||||
"severity": "HIGH",
|
||||
"evidence": "Obfusc: L393: co = marshal.load(fp) | L395: trace(f\"_read_pyc({source}): marshal.load error {e}\")\nExec: L188: exec(co, module.__dict__)"
|
||||
},
|
||||
{
|
||||
"package": "scikit-learn",
|
||||
"file": "sklearn/externals/array_api_compat/torch/__init__.py",
|
||||
|
|
@ -1318,6 +1290,13 @@
|
|||
"severity": "HIGH",
|
||||
"evidence": "Obfusc: L3078: module = __import__('transformers', fromlist=[model_class_name])\nExec: L2960: exec(f\"from transformers.modeling_utils import ({', '.join(functions)})\", locals(), globals()) | L3006: exec(save_pretrained, globals(), functions)"
|
||||
},
|
||||
{
|
||||
"package": "unsloth-zoo",
|
||||
"file": "tests/test_mlx_trainer_internals.py",
|
||||
"check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval",
|
||||
"severity": "HIGH",
|
||||
"evidence": "Obfusc: L430: assert ppl == pytest.approx(__import__(\"math\").exp(2.5))\nExec: L408: def eval(self):"
|
||||
},
|
||||
{
|
||||
"package": "werkzeug",
|
||||
"file": "werkzeug/routing/rules.py",
|
||||
|
|
|
|||
|
|
@ -11,10 +11,12 @@ peft==0.18.1
|
|||
|
||||
# TRL and related packages
|
||||
trl==0.23.1
|
||||
git+https://github.com/meta-pytorch/OpenEnv.git
|
||||
# executorch>=1.0.1 # 41.5 MB - no imports in unsloth/zoo/studio
|
||||
torch-c-dlpack-ext
|
||||
sentence_transformers==5.2.0
|
||||
transformers==4.57.6
|
||||
pytorch_tokenizers
|
||||
kernels==0.12.1
|
||||
# kernels<3.11 imports tomli as its tomllib fallback; --no-deps skips its own
|
||||
# marker dep, so list it here (no-op on the 3.12/3.13 default installs).
|
||||
tomli; python_version < "3.11"
|
||||
|
|
|
|||
|
|
@ -1,27 +1,11 @@
|
|||
# OpenEnv dependencies
|
||||
tomli
|
||||
tomli-w
|
||||
|
||||
# ExecuTorch dependencies
|
||||
ruamel.yaml
|
||||
# coremltools # 10.2 MB - Apple CoreML, no imports in unsloth/zoo/studio
|
||||
expecttest
|
||||
# transitive dep of onnxruntime (via data-designer's pymupdf4llm)
|
||||
flatbuffers
|
||||
hydra-core
|
||||
hypothesis
|
||||
kgb
|
||||
parameterized
|
||||
pytest>=9.0.3,<10
|
||||
pytest-json-report
|
||||
pytest-rerunfailures>=16.2,<17
|
||||
pytest-xdist
|
||||
# Also needed by sentence_transformers (installed with --no-deps in extras-no-deps.txt)
|
||||
# Also needed by sentence_transformers (installed with --no-deps in extras-no-deps.txt);
|
||||
# librosa pulls it in too, but is skipped in no-torch mode.
|
||||
scikit-learn==1.7.1
|
||||
|
||||
# Additional extras
|
||||
pybind11
|
||||
langid
|
||||
jiwer
|
||||
jiwer # WER/CER metrics for vision OCR save-merge benchmarks
|
||||
omegaconf
|
||||
einx
|
||||
pyloudnorm
|
||||
|
|
@ -44,12 +28,7 @@ pystoi
|
|||
soundfile
|
||||
tensorboard
|
||||
torch-stoi
|
||||
evaluate
|
||||
timm
|
||||
transformers-cfg
|
||||
open_spiel
|
||||
addict
|
||||
easydict
|
||||
einops
|
||||
tabulate
|
||||
openai>=2.7.2
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ huggingface-hub==0.36.2
|
|||
datasets==4.3.0
|
||||
pyarrow==23.0.1
|
||||
|
||||
# FastMCP/OpenEnv compat
|
||||
# FastMCP compat
|
||||
fastmcp>=3.0.2
|
||||
mcp>=1.24,<2
|
||||
websockets>=15.0.1
|
||||
|
|
|
|||
|
|
@ -9,8 +9,6 @@ pandas
|
|||
nest_asyncio
|
||||
datasets==4.3.0
|
||||
pyjwt
|
||||
easydict
|
||||
addict
|
||||
# gradio>=4.0.0 # 148 MB - Studio uses React + FastAPI, not Gradio
|
||||
huggingface-hub==0.36.2
|
||||
structlog>=24.1.0
|
||||
|
|
|
|||
|
|
@ -1658,7 +1658,7 @@ def run(
|
|||
|
||||
|
||||
# Packages to skip on Windows (require special build steps)
|
||||
WINDOWS_SKIP_PACKAGES = {"open_spiel", "triton_kernels"}
|
||||
WINDOWS_SKIP_PACKAGES = {"triton_kernels"}
|
||||
|
||||
# Packages to skip when torch is unavailable (Intel Mac GGUF-only mode).
|
||||
# These either *are* torch extensions or have unconditional
|
||||
|
|
@ -1675,7 +1675,6 @@ NO_TORCH_SKIP_PACKAGES = {
|
|||
"torchcodec",
|
||||
"torch-c-dlpack-ext",
|
||||
"openai-whisper",
|
||||
"transformers-cfg",
|
||||
"librosa",
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -866,13 +866,11 @@ class TestInstallPythonStackFiltering:
|
|||
|
||||
result_path = ips._filter_requirements(extras, ips.NO_TORCH_SKIP_PACKAGES)
|
||||
filtered = Path(result_path).read_text(encoding = "utf-8").lower()
|
||||
lines = [
|
||||
l.strip() for l in filtered.splitlines() if l.strip() and not l.strip().startswith("#")
|
||||
]
|
||||
|
||||
for pkg in ["torch-stoi", "timm", "openai-whisper", "transformers-cfg"]:
|
||||
lines = [
|
||||
l.strip()
|
||||
for l in filtered.splitlines()
|
||||
if l.strip() and not l.strip().startswith("#")
|
||||
]
|
||||
for pkg in ips.NO_TORCH_SKIP_PACKAGES:
|
||||
assert not any(
|
||||
l.startswith(pkg) for l in lines
|
||||
), f"{pkg} should be removed from extras.txt"
|
||||
|
|
|
|||
|
|
@ -138,7 +138,6 @@ class TestFilterRequirements:
|
|||
req = self._write_req(
|
||||
tmp_path,
|
||||
"""\
|
||||
open_spiel
|
||||
triton_kernels
|
||||
torch-stoi
|
||||
timm
|
||||
|
|
@ -222,8 +221,8 @@ class TestRealRequirementsFiltering:
|
|||
filtered = self._non_blank_non_comment(Path(result))
|
||||
original = self._non_blank_non_comment(EXTRAS_TXT)
|
||||
|
||||
# These must be gone.
|
||||
for pkg in ["torch-stoi", "timm", "openai-whisper", "transformers-cfg"]:
|
||||
# Every NO_TORCH skip package present in extras.txt must be gone.
|
||||
for pkg in ips.NO_TORCH_SKIP_PACKAGES:
|
||||
assert not any(
|
||||
l.lower().startswith(pkg) for l in filtered
|
||||
), f"{pkg} should be removed from extras.txt"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue