From 1a274c488e9621281f86e2f6e85316a77e2c8070 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 8 Jul 2026 07:51:53 -0700 Subject: [PATCH 001/372] Bump install.sh / install.ps1 pins to unsloth>=2026.7.2 and unsloth-zoo>=2026.7.2 (#6981) PyPI release unsloth 2026.7.2 is now live. Bumps the pinned floor in install.sh and install.ps1 from 2026.7.1 to 2026.7.2 for both unsloth and unsloth-zoo across all 5 install commands (no-torch / reinstall / upgrade / local / auto torch backend paths) so fresh installs resolve to the new wheel. Follows the same pattern as #5716. --- install.ps1 | 10 +++++----- install.sh | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/install.ps1 b/install.ps1 index 9114f80af9..696f4e613a 100644 --- a/install.ps1 +++ b/install.ps1 @@ -2155,7 +2155,7 @@ exit 0 if ($SkipTorch) { # No-torch: install unsloth + unsloth-zoo with --no-deps, then # runtime deps (typer, safetensors, transformers, etc.) with --no-deps. - $baseInstallExit = Invoke-InstallCommandRetry -Label "install unsloth (migrated no-torch)" { uv pip install --python $VenvPython --no-deps --reinstall-package unsloth --reinstall-package unsloth-zoo "unsloth>=2026.7.1" "unsloth-zoo>=2026.7.1" } + $baseInstallExit = Invoke-InstallCommandRetry -Label "install unsloth (migrated no-torch)" { uv pip install --python $VenvPython --no-deps --reinstall-package unsloth --reinstall-package unsloth-zoo "unsloth>=2026.7.2" "unsloth-zoo>=2026.7.2" } if ($baseInstallExit -eq 0) { # Resolve pydantic WITH deps so pip pins pydantic-core # to the matching version (no-torch-runtime.txt below @@ -2169,7 +2169,7 @@ exit 0 } } } else { - $baseInstallExit = Invoke-InstallCommandRetry -Label "install unsloth (migrated)" { uv pip install --python $VenvPython --reinstall-package unsloth --reinstall-package unsloth-zoo "unsloth>=2026.7.1" "unsloth-zoo>=2026.7.1" } + $baseInstallExit = Invoke-InstallCommandRetry -Label "install unsloth (migrated)" { uv pip install --python $VenvPython --reinstall-package unsloth --reinstall-package unsloth-zoo "unsloth>=2026.7.2" "unsloth-zoo>=2026.7.2" } } if ($baseInstallExit -ne 0) { Write-Host "[ERROR] Failed to install unsloth (exit code $baseInstallExit)" -ForegroundColor Red @@ -2235,7 +2235,7 @@ exit 0 if ($SkipTorch) { # No-torch: install unsloth + unsloth-zoo with --no-deps, then # runtime deps (typer, safetensors, transformers, etc.) with --no-deps. - $baseInstallExit = Invoke-InstallCommandRetry -Label "install unsloth (no-torch)" { uv pip install --python $VenvPython --no-deps --upgrade-package unsloth --upgrade-package unsloth-zoo "unsloth>=2026.7.1" "unsloth-zoo>=2026.7.1" } + $baseInstallExit = Invoke-InstallCommandRetry -Label "install unsloth (no-torch)" { uv pip install --python $VenvPython --no-deps --upgrade-package unsloth --upgrade-package unsloth-zoo "unsloth>=2026.7.2" "unsloth-zoo>=2026.7.2" } if ($baseInstallExit -eq 0) { # Same pydantic-with-deps trick as the migrated branch. $baseInstallExit = Invoke-InstallCommandRetry -Label "install pydantic" { uv pip install --python $VenvPython pydantic } @@ -2247,7 +2247,7 @@ exit 0 } } } elseif ($StudioLocalInstall) { - $baseInstallExit = Invoke-InstallCommandRetry -Label "install unsloth (local)" { uv pip install --python $VenvPython --upgrade-package unsloth "unsloth>=2026.7.1" "unsloth-zoo>=2026.7.1" } + $baseInstallExit = Invoke-InstallCommandRetry -Label "install unsloth (local)" { uv pip install --python $VenvPython --upgrade-package unsloth "unsloth>=2026.7.2" "unsloth-zoo>=2026.7.2" } } else { $baseInstallExit = Invoke-InstallCommandRetry -Label "install unsloth" { uv pip install --python $VenvPython --upgrade-package unsloth -- "$PackageName" } } @@ -2275,7 +2275,7 @@ exit 0 Write-TauriLog "STEP" "Installing unsloth" substep "installing unsloth (this may take a few minutes)..." if ($StudioLocalInstall) { - $baseInstallExit = Invoke-InstallCommandRetry -Label "install unsloth (auto torch backend)" { uv pip install --python $VenvPython "unsloth-zoo>=2026.7.1" "unsloth>=2026.7.1" --torch-backend=auto } + $baseInstallExit = Invoke-InstallCommandRetry -Label "install unsloth (auto torch backend)" { uv pip install --python $VenvPython "unsloth-zoo>=2026.7.2" "unsloth>=2026.7.2" --torch-backend=auto } if ($baseInstallExit -ne 0) { Write-Host "[ERROR] Failed to install unsloth (exit code $baseInstallExit)" -ForegroundColor Red return (Exit-InstallFailure "Failed to install unsloth (exit code $baseInstallExit)" $baseInstallExit) diff --git a/install.sh b/install.sh index 796d80e401..0acc9ec0be 100755 --- a/install.sh +++ b/install.sh @@ -2706,7 +2706,7 @@ if [ "$_MIGRATED" = true ]; then # to prevent transitive torch resolution. run_install_cmd_retry "install unsloth (migrated no-torch)" uv pip install --python "$_VENV_PY" --no-deps \ --reinstall-package unsloth --reinstall-package unsloth-zoo \ - "unsloth>=2026.7.1" "unsloth-zoo>=2026.7.1" + "unsloth>=2026.7.2" "unsloth-zoo>=2026.7.2" # Resolve pydantic WITH deps so pip pins pydantic-core to the # matching version (no-torch-runtime.txt below is --no-deps). # All transitive deps are torch-free. @@ -2721,7 +2721,7 @@ if [ "$_MIGRATED" = true ]; then # overrides file, so UV_OVERRIDE is unset and this positional is the only cover. run_install_cmd_retry "install unsloth (migrated)" uv pip install --python "$_VENV_PY" \ --reinstall-package unsloth --reinstall-package unsloth-zoo \ - "unsloth>=2026.7.1" "unsloth-zoo>=2026.7.1" ${_MLX_LM_EXCLUDE_ARG:-} + "unsloth>=2026.7.2" "unsloth-zoo>=2026.7.2" ${_MLX_LM_EXCLUDE_ARG:-} fi if [ "$STUDIO_LOCAL_INSTALL" = true ]; then substep "overlaying local repo (editable)..." @@ -2925,7 +2925,7 @@ elif [ -n "$TORCH_INDEX_URL" ]; then # runtime deps (typer, safetensors, transformers, etc.) with --no-deps. run_install_cmd_retry "install unsloth (no-torch)" uv pip install --python "$_VENV_PY" --no-deps \ --upgrade-package unsloth --upgrade-package unsloth-zoo \ - "unsloth>=2026.7.1" "unsloth-zoo>=2026.7.1" + "unsloth>=2026.7.2" "unsloth-zoo>=2026.7.2" # Same pydantic-with-deps trick as the migrated branch. run_install_cmd_retry "install pydantic (with deps for compatible core)" \ uv pip install --python "$_VENV_PY" pydantic @@ -2943,7 +2943,7 @@ elif [ -n "$TORCH_INDEX_URL" ]; then fi elif [ "$STUDIO_LOCAL_INSTALL" = true ]; then run_install_cmd_retry "install unsloth (local)" uv pip install --python "$_VENV_PY" \ - --upgrade-package unsloth "unsloth>=2026.7.1" "unsloth-zoo>=2026.7.1" + --upgrade-package unsloth "unsloth>=2026.7.2" "unsloth-zoo>=2026.7.2" substep "overlaying local repo (editable)..." run_install_cmd "overlay local repo" uv pip install --python "$_VENV_PY" -e "$_REPO_ROOT" --no-deps substep "overlaying unsloth-zoo from git main..." @@ -2975,7 +2975,7 @@ else tauri_log "STEP" "Installing Unsloth" substep "installing unsloth (this may take a few minutes)..." if [ "$STUDIO_LOCAL_INSTALL" = true ]; then - run_install_cmd_retry "install unsloth (auto torch backend)" uv pip install --python "$_VENV_PY" "unsloth-zoo>=2026.7.1" "unsloth>=2026.7.1" --torch-backend=auto + run_install_cmd_retry "install unsloth (auto torch backend)" uv pip install --python "$_VENV_PY" "unsloth-zoo>=2026.7.2" "unsloth>=2026.7.2" --torch-backend=auto substep "overlaying local repo (editable)..." run_install_cmd "overlay local repo" uv pip install --python "$_VENV_PY" -e "$_REPO_ROOT" --no-deps substep "overlaying unsloth-zoo from git main..." From 5c2e53606e513cab3b698852b928b2b4484ada76 Mon Sep 17 00:00:00 2001 From: Michael Han <107991372+shimmyshimmer@users.noreply.github.com> Date: Wed, 8 Jul 2026 08:14:03 -0700 Subject: [PATCH 002/372] Studio: render thinking blocks for safetensors inference with prefilled templates (#6816) * Studio: render thinking blocks for safetensors inference with prefilled templates Reasoning templates like Qwen3.6 end the generation prompt with an open tag. skip_prompt streaming drops it, so the frontend never sees the opening tag and shows reasoning as plain text. Detect the prefill and re-emit it at the start of the stream on the transformers and MLX paths. Also stop stripping think tags in _clean_generated_text when a tokenizer marks them special. * Studio: guard think re-emit for special close tags, yield prefill early Address review feedback: - Guard: skip re-emitting the open when the tokenizer marks as a special token, since skip_special_tokens would strip the model's close tag and leave an unclosed block that swallows the answer. Falls back to plain text (pre-fix behaviour) for those tokenizers. - Yield the prefilled before the first token so the thinking block renders during prompt prefill instead of after the first generated token. - Drop the now-unnecessary _clean_generated_text think-tag exemption; the guard handles the special-token case at the source. No mainstream reasoning model (Qwen3.6, Qwen3, DeepSeek-R1, QwQ, GLM-4.6) marks think tags special, so behaviour is unchanged for them. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Lyxot --- .../core/inference/chat_template_helpers.py | 37 ++++++++ studio/backend/core/inference/inference.py | 31 ++++++- .../backend/core/inference/mlx_inference.py | 20 ++++- .../tests/test_think_prefill_reemit.py | 89 +++++++++++++++++++ 4 files changed, 173 insertions(+), 4 deletions(-) create mode 100644 studio/backend/tests/test_think_prefill_reemit.py diff --git a/studio/backend/core/inference/chat_template_helpers.py b/studio/backend/core/inference/chat_template_helpers.py index dfd4c1c0bc..897db8262d 100644 --- a/studio/backend/core/inference/chat_template_helpers.py +++ b/studio/backend/core/inference/chat_template_helpers.py @@ -12,6 +12,43 @@ import json import logging from typing import Optional +_THINK_OPEN = "" +_THINK_CLOSE = "" + + +def detect_think_prefill(prompt: Optional[str], special_tokens = None) -> str: + """Return the trailing open ```` prefill of a rendered prompt. + + Reasoning templates (Qwen3.6, DeepSeek-R1-style) end the generation + prompt with ``\\n`` so the model starts reasoning immediately. + Because that opening tag is part of the *prompt*, skip_prompt streaming + never emits it, and the frontend's ````/```` parser shows + the reasoning as plain text instead of a thinking block. (The GGUF path + is unaffected: llama-server's reasoning parser returns + ``reasoning_content``, which gets re-wrapped in think tags.) + + Returns the exact prompt tail to re-emit at the start of the generated + stream (e.g. ``"\\n"``), or ``""`` when the prompt does not end + with an open think block, including the ``enable_thinking=False`` case + where templates prefill an already-closed ``\\n\\n``. + + ``special_tokens`` is the tokenizer's special-token list. If ```` + is one, the streamer's skip_special_tokens strips the model's closing tag, + so re-emitting the open would leave an unclosed block that swallows the + answer. In that case return ``""`` and fall back to plain text. + """ + if not prompt: + return "" + open_idx = prompt.rfind(_THINK_OPEN) + if open_idx == -1: + return "" + tail = prompt[open_idx:] + if _THINK_CLOSE in tail or tail.strip() != _THINK_OPEN: + return "" + if special_tokens and _THINK_CLOSE in set(special_tokens): + return "" + return tail + logger = logging.getLogger(__name__) diff --git a/studio/backend/core/inference/inference.py b/studio/backend/core/inference/inference.py index 167706f701..7e69e05124 100644 --- a/studio/backend/core/inference/inference.py +++ b/studio/backend/core/inference/inference.py @@ -1178,13 +1178,22 @@ class InferenceBackend: add_special_tokens = False, return_tensors = "pt", ).to(model.device) + prompt_text = input_text else: # Text-only path for a vision model formatted_prompt = self.format_chat_prompt(messages, system_prompt) inputs = raw_tokenizer(formatted_prompt, return_tensors = "pt").to(model.device) + prompt_text = formatted_prompt # Stream with TextIteratorStreamer + background thread try: + from core.inference.chat_template_helpers import detect_think_prefill + + # Re-emit an open prefill swallowed by skip_prompt (see + # generate_stream). + think_prefix = detect_think_prefill( + prompt_text, getattr(raw_tokenizer, "all_special_tokens", None) + ) from transformers import TextIteratorStreamer import threading @@ -1233,7 +1242,11 @@ class InferenceBackend: thread = threading.Thread(target = generate_fn) thread.start() - output = "" + output = think_prefix + # Emit the prefilled before the first token so the block + # renders during prompt prefill (which can take seconds). + if think_prefix: + yield think_prefix from queue import Empty generation_complete = False @@ -1467,6 +1480,16 @@ class InferenceBackend: from transformers import TextIteratorStreamer import threading + from core.inference.chat_template_helpers import detect_think_prefill + + # skip_prompt swallows an open prefilled by the template; + # re-emit it so the frontend can render the thinking block. + # gpt-oss emits its own tags via HarmonyTextStreamer. + think_prefix = ( + "" + if self._is_gpt_oss_model() + else detect_think_prefill(prompt, getattr(tokenizer, "all_special_tokens", None)) + ) # gpt-oss models: HarmonyTextStreamer parses the multi-channel # harmony protocol into tags @@ -1550,7 +1573,11 @@ class InferenceBackend: thread = threading.Thread(target = generate_fn) thread.start() - output = "" + output = think_prefix + # Emit the prefilled before the first token so the block + # renders during prompt prefill (which can take seconds). + if think_prefix: + yield think_prefix from queue import Empty generation_complete = False diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.py index d84baa278d..6287b184a6 100644 --- a/studio/backend/core/inference/mlx_inference.py +++ b/studio/backend/core/inference/mlx_inference.py @@ -485,6 +485,7 @@ class MLXInferenceBackend: from core.inference.chat_template_helpers import ( apply_chat_template_for_generation, + detect_think_prefill, render_with_native_template_fallback, ) @@ -518,6 +519,15 @@ class MLXInferenceBackend: hf_token = model_info.get("hf_token"), ) + # An open prefilled by the template lives in the prompt, not + # the generated tokens; re-emit it so the frontend renders the block. + think_prefix = detect_think_prefill( + prompt, getattr(self._tokenizer, "all_special_tokens", None) + ) + # Emit it before the first token so the block renders during prefill. + if think_prefix: + yield think_prefix + sampler = make_sampler( temp = temperature, top_p = top_p, @@ -570,7 +580,7 @@ class MLXInferenceBackend: token_ids, skip_special_tokens = True, ) - yield cumulative + yield think_prefix + cumulative if cancel_event and cancel_event.is_set(): break @@ -634,7 +644,13 @@ class MLXInferenceBackend: # mlx_vlm's stream_generate handles pixel_values (None for text-only) images = [image] if image is not None else None - cumulative = "" + from core.inference.chat_template_helpers import detect_think_prefill + + # Re-emit an open prefill from the prompt (see _generate_text). + cumulative = detect_think_prefill(prompt, getattr(chat_target, "all_special_tokens", None)) + # Emit it before the first token so the block renders during prefill. + if cumulative: + yield cumulative logger.info( "VLM generating: prompt_len=%d, has_image=%s", len(prompt), diff --git a/studio/backend/tests/test_think_prefill_reemit.py b/studio/backend/tests/test_think_prefill_reemit.py new file mode 100644 index 0000000000..300ff92776 --- /dev/null +++ b/studio/backend/tests/test_think_prefill_reemit.py @@ -0,0 +1,89 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +""" +Unit tests for detect_think_prefill. + +Reasoning templates (Qwen3.6-style) end the generation prompt with an open +``\\n`` so the model starts reasoning immediately. skip_prompt +streaming drops that opening tag, so the safetensors/MLX paths must re-emit +it for the frontend's parser to render a thinking block. +""" + +import os +import sys + +_backend = os.path.join(os.path.dirname(__file__), "..") +sys.path.insert(0, _backend) + +from core.inference.chat_template_helpers import detect_think_prefill + + +QWEN_PROMPT = "<|im_start|>user\nHi!<|im_end|>\n<|im_start|>assistant\n" + + +def test_open_think_prefill_reemitted(): + """Qwen3.6-style enable_thinking=True prompt tail: \\n.""" + assert detect_think_prefill(QWEN_PROMPT + "\n") == "\n" + + +def test_bare_open_think_prefill_reemitted(): + """Prefill without trailing newline still detected.""" + assert detect_think_prefill(QWEN_PROMPT + "") == "" + + +def test_closed_think_prefill_not_reemitted(): + """enable_thinking=False prefills a closed, empty think block.""" + assert detect_think_prefill(QWEN_PROMPT + "\n\n\n\n") == "" + + +def test_prompt_without_think_untouched(): + """Non-reasoning templates produce no prefix.""" + assert detect_think_prefill(QWEN_PROMPT) == "" + + +def test_historical_think_blocks_ignored(): + """A closed think block in a prior assistant turn (preserve_thinking) + must not trigger re-emission when the generation tail is plain.""" + prompt = ( + "<|im_start|>user\nHi!<|im_end|>\n" + "<|im_start|>assistant\n\nprior reasoning\n\n\nHello!<|im_end|>\n" + "<|im_start|>user\nAgain?<|im_end|>\n<|im_start|>assistant\n" + ) + assert detect_think_prefill(prompt) == "" + + +def test_historical_blocks_plus_open_prefill(): + """Prior closed blocks plus a fresh open prefill: only the tail matters.""" + prompt = ( + "<|im_start|>assistant\n\nprior\n\n\nHello!<|im_end|>\n" + "<|im_start|>assistant\n\n" + ) + assert detect_think_prefill(prompt) == "\n" + + +def test_content_after_open_tag_not_reemitted(): + """If non-whitespace follows the tag it is not a plain prefill.""" + assert detect_think_prefill(QWEN_PROMPT + "\npartial reasoning") == "" + + +def test_empty_and_none_prompts(): + assert detect_think_prefill("") == "" + assert detect_think_prefill(None) == "" + + +def test_guard_suppresses_when_close_tag_is_special(): + """If is a special token, skip_special_tokens strips the model's + close tag, so re-emitting the open would leave an unclosed block. Guard off.""" + specials = ["<|im_end|>", "", ""] + assert detect_think_prefill(QWEN_PROMPT + "\n", specials) == "" + + +def test_guard_emits_when_think_not_special(): + specials = ["<|im_end|>", "<|endoftext|>"] + assert detect_think_prefill(QWEN_PROMPT + "\n", specials) == "\n" + + +def test_guard_default_and_empty_keep_emitting(): + assert detect_think_prefill(QWEN_PROMPT + "\n", None) == "\n" + assert detect_think_prefill(QWEN_PROMPT + "\n", []) == "\n" From 7a9fb4404e5c81ef5eb34de7d7944bd476c6192b Mon Sep 17 00:00:00 2001 From: Michael Han <107991372+shimmyshimmer@users.noreply.github.com> Date: Wed, 8 Jul 2026 08:17:09 -0700 Subject: [PATCH 003/372] Remove API menu new badge (#6983) --- studio/frontend/src/components/app-sidebar.tsx | 3 --- 1 file changed, 3 deletions(-) diff --git a/studio/frontend/src/components/app-sidebar.tsx b/studio/frontend/src/components/app-sidebar.tsx index f59a952b3a..823e420869 100644 --- a/studio/frontend/src/components/app-sidebar.tsx +++ b/studio/frontend/src/components/app-sidebar.tsx @@ -1572,9 +1572,6 @@ export function AppSidebar() { > {t("shell.navigation.api")} - - {t("common.new")} - } From 92c3e48529c8b7f96033f52f93845819b2ae53e3 Mon Sep 17 00:00:00 2001 From: Vineeth Sai Date: Wed, 8 Jul 2026 13:37:44 -0700 Subject: [PATCH 004/372] Fix BAD_MAPPINGS not redirecting the -unsloth-bnb-4bit dynamic quants (#6949) --------- Co-authored-by: oobabooga <112222186+oobabooga@users.noreply.github.com> --- .github/workflows/consolidated-tests-ci.yml | 1 + tests/test_bad_mappings_redirect.py | 47 +++++++++++++++++++++ unsloth/models/loader_utils.py | 5 +++ 3 files changed, 53 insertions(+) create mode 100644 tests/test_bad_mappings_redirect.py diff --git a/.github/workflows/consolidated-tests-ci.yml b/.github/workflows/consolidated-tests-ci.yml index ae4b386589..6ff3d19ba2 100644 --- a/.github/workflows/consolidated-tests-ci.yml +++ b/.github/workflows/consolidated-tests-ci.yml @@ -364,6 +364,7 @@ jobs: tests/utils/test_attention_masks.py \ tests/utils/test_trunc_normal_patch.py \ tests/python/test_fast_language_model_text_only.py \ + tests/test_bad_mappings_redirect.py \ tests/test_prefetch_snapshot_scope.py \ --deselect 'tests/utils/test_attention_masks.py::test_run_attention_flash_varlen_receives_window_and_softcap' # The deselected test monkeypatches flash_attn_varlen_func, which is diff --git a/tests/test_bad_mappings_redirect.py b/tests/test_bad_mappings_redirect.py new file mode 100644 index 0000000000..49dab2d98b --- /dev/null +++ b/tests/test_bad_mappings_redirect.py @@ -0,0 +1,47 @@ +"""Regression test for BAD_MAPPINGS redirecting oversized dynamic quants. + +get_model_name previously applied BAD_MAPPINGS only to the resolver's output, +but several listed names (the `-unsloth-bnb-4bit` dynamic quants, plus any name +the resolver doesn't map) come back as None, so their BAD_MAPPINGS entries were +dead and the oversized model loaded. Asserting over every entry catches all of +them. The mapper table and the resolver have no heavy imports of their own, +so we exec the import-free mapper module and ast-extract the resolver functions +rather than importing unsloth (which needs a GPU). +""" + +import ast +import os + +_MODELS = os.path.join(os.path.dirname(__file__), os.pardir, "unsloth", "models") + + +def _load_get_model_name(): + mapper_ns = {} + with open(os.path.join(_MODELS, "mapper.py"), encoding = "utf-8") as f: + exec(compile(f.read(), "mapper.py", "exec"), mapper_ns) + + with open(os.path.join(_MODELS, "loader_utils.py"), encoding = "utf-8") as f: + tree = ast.parse(f.read()) + + namespace = dict(mapper_ns) + namespace["SUPPORTS_FOURBIT"] = True + namespace["_env_says_offline"] = lambda: True + namespace["_get_new_mapper"] = lambda: ({}, {}, {}) + + wanted = {"__get_model_name", "_resolve_with_mappers", "get_model_name"} + for node in tree.body: + if isinstance(node, ast.Assign) and any( + getattr(target, "id", None) == "BAD_MAPPINGS" for target in node.targets + ): + exec(compile(ast.Module([node], []), "", "exec"), namespace) + elif isinstance(node, ast.FunctionDef) and node.name in wanted: + exec(compile(ast.Module([node], []), node.name, "exec"), namespace) + + return namespace["get_model_name"], namespace["BAD_MAPPINGS"] + + +def test_bad_mappings_redirect_every_listed_name(): + get_model_name, bad_mappings = _load_get_model_name() + assert bad_mappings, "BAD_MAPPINGS should not be empty" + for name, expected in bad_mappings.items(): + assert get_model_name(name, load_in_4bit = True) == expected, name diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py index d6d6ce877e..fa6282bcf6 100644 --- a/unsloth/models/loader_utils.py +++ b/unsloth/models/loader_utils.py @@ -239,6 +239,11 @@ def get_model_name( and new_model_name.lower() in BAD_MAPPINGS ): new_model_name = BAD_MAPPINGS[new_model_name.lower()] + elif new_model_name is None and model_name.lower() in BAD_MAPPINGS: + # Some bad names (e.g. the `-unsloth-bnb-4bit` dynamic quants) are keys + # of the mappers, not values, so the resolver returns None for them and + # the remap above is skipped; remap the input name directly instead. + new_model_name = BAD_MAPPINGS[model_name.lower()] if ( new_model_name is None From dc4618ce475554d99fab76daae9746baa62bf090 Mon Sep 17 00:00:00 2001 From: Vineeth Sai Date: Wed, 8 Jul 2026 13:40:39 -0700 Subject: [PATCH 005/372] Fix duplicate unsloth/gemma-2b-bnb-4bit mapper key routing the base 4bit repo to the instruct model (#6891) --- .github/workflows/consolidated-tests-ci.yml | 1 + tests/test_gemma_2b_mapper_key.py | 46 +++++++++++++++++++++ unsloth/models/mapper.py | 2 +- 3 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 tests/test_gemma_2b_mapper_key.py diff --git a/.github/workflows/consolidated-tests-ci.yml b/.github/workflows/consolidated-tests-ci.yml index 6ff3d19ba2..1bb4c2bb58 100644 --- a/.github/workflows/consolidated-tests-ci.yml +++ b/.github/workflows/consolidated-tests-ci.yml @@ -366,6 +366,7 @@ jobs: tests/python/test_fast_language_model_text_only.py \ tests/test_bad_mappings_redirect.py \ tests/test_prefetch_snapshot_scope.py \ + tests/test_gemma_2b_mapper_key.py \ --deselect 'tests/utils/test_attention_masks.py::test_run_attention_flash_varlen_receives_window_and_softcap' # The deselected test monkeypatches flash_attn_varlen_func, which is # only bound on the module when `flash_attn` is importable. flash_attn diff --git a/tests/test_gemma_2b_mapper_key.py b/tests/test_gemma_2b_mapper_key.py new file mode 100644 index 0000000000..31edacfce4 --- /dev/null +++ b/tests/test_gemma_2b_mapper_key.py @@ -0,0 +1,46 @@ +"""Regression test for the duplicate ``unsloth/gemma-2b-bnb-4bit`` key in +``unsloth/models/mapper.py``. + +The 4bit instruction-tuned Gemma 2B entry was accidentally keyed with the base +model's repo name, so ``__INT_TO_FLOAT_MAPPER`` held two identical +``unsloth/gemma-2b-bnb-4bit`` keys. Python keeps only the last value for a +duplicate literal key, so the base 4bit repo resolved to the *instruct* model, +the base model lost its reverse (4x-faster) mapping, and +``unsloth/gemma-2b-it-bnb-4bit`` was never registered at all. + +``mapper.py`` has no imports, so we exec it directly and inspect the built +mappers without importing ``unsloth`` (which requires a GPU). +""" + +import os + +MAPPER_PATH = os.path.join(os.path.dirname(__file__), os.pardir, "unsloth", "models", "mapper.py") + + +def _load_mappers(): + with open(MAPPER_PATH) as f: + source = f.read() + namespace = {} + exec(compile(source, MAPPER_PATH, "exec"), namespace) + return namespace + + +def test_gemma_2b_base_and_instruct_4bit_are_distinct(): + namespace = _load_mappers() + int_to_float = namespace["INT_TO_FLOAT_MAPPER"] + float_to_int = namespace["FLOAT_TO_INT_MAPPER"] + + # The base 4bit repo must resolve to the base model, not the instruct one. + assert int_to_float["unsloth/gemma-2b-bnb-4bit"] == "unsloth/gemma-2b" + + # The instruct 4bit repo must be registered and resolve to the instruct model. + assert "unsloth/gemma-2b-it-bnb-4bit" in int_to_float + assert int_to_float["unsloth/gemma-2b-it-bnb-4bit"] == "unsloth/gemma-2b-it" + + # The base model must reverse-map back to the base 4bit repo. + assert float_to_int["unsloth/gemma-2b"] == "unsloth/gemma-2b-bnb-4bit" + assert float_to_int["google/gemma-2b"] == "unsloth/gemma-2b-bnb-4bit" + + # The instruct model must reverse-map to the instruct 4bit repo. + assert float_to_int["unsloth/gemma-2b-it"] == "unsloth/gemma-2b-it-bnb-4bit" + assert float_to_int["google/gemma-2b-it"] == "unsloth/gemma-2b-it-bnb-4bit" diff --git a/unsloth/models/mapper.py b/unsloth/models/mapper.py index 57c1e292c3..f3a0e1f9bb 100644 --- a/unsloth/models/mapper.py +++ b/unsloth/models/mapper.py @@ -134,7 +134,7 @@ __INT_TO_FLOAT_MAPPER = \ "unsloth/gemma-7b-it", "google/gemma-7b-it", ), - "unsloth/gemma-2b-bnb-4bit" : ( + "unsloth/gemma-2b-it-bnb-4bit" : ( "unsloth/gemma-2b-it", "google/gemma-2b-it", ), From 85a068cfe10f2f8bc214dbd2e0b82164419fc8de Mon Sep 17 00:00:00 2001 From: Vineeth Sai Date: Wed, 8 Jul 2026 13:57:41 -0700 Subject: [PATCH 006/372] Fix to_sharegpt optional block rendering "None" for missing extra columns (#6827) --------- Co-authored-by: oobabooga <112222186+oobabooga@users.noreply.github.com> --- .../python/test_to_sharegpt_optional_none.py | 97 +++++++++++++++++++ unsloth/chat_templates.py | 13 ++- 2 files changed, 108 insertions(+), 2 deletions(-) create mode 100644 tests/python/test_to_sharegpt_optional_none.py diff --git a/tests/python/test_to_sharegpt_optional_none.py b/tests/python/test_to_sharegpt_optional_none.py new file mode 100644 index 0000000000..05fcb8a6a8 --- /dev/null +++ b/tests/python/test_to_sharegpt_optional_none.py @@ -0,0 +1,97 @@ +import ast +import re +from pathlib import Path + + +def _load_formatter_builders(): + # Extract _parse_combined_prompt and _create_formatter without importing + # unsloth (importing unsloth needs unsloth_zoo / a GPU). Both are pure + # Python and only use the `re` module. + source = Path(__file__).parents[2] / "unsloth" / "chat_templates.py" + tree = ast.parse(source.read_text(encoding = "utf-8")) + wanted = {"_parse_combined_prompt", "_create_formatter"} + funcs = [ + node for node in tree.body if isinstance(node, ast.FunctionDef) and node.name in wanted + ] + namespace = {"re": re} + module = ast.Module(body = funcs, type_ignores = []) + ast.fix_missing_locations(module) + exec(compile(module, str(source), "exec"), namespace) + return namespace["_parse_combined_prompt"], namespace["_create_formatter"] + + +class _StubDataset: + def __init__(self, column_names): + self.column_names = column_names + + +def _render(merged_prompt, columns, batch): + parse, create = _load_formatter_builders() + possible_columns, final_optional_prompts = parse(merged_prompt, _StubDataset(columns)) + processor = create(possible_columns, final_optional_prompts, "text") + return processor(batch)["text"] + + +def test_optional_block_missing_second_column_does_not_render_none(): + # A [[...]] block may reference several columns; only the first gates the + # block. A later column that is None must not render as the literal "None". + merged_prompt = "Location: [[{city}, {country}]] end" + out = _render( + merged_prompt, + ["city", "country"], + {"city": ["Paris"], "country": [None]}, + ) + assert out[0] == "Location: Paris, end" + assert "None" not in out[0] + + +def test_optional_block_all_columns_present_unchanged(): + merged_prompt = "Location: [[{city}, {country}]] end" + out = _render( + merged_prompt, + ["city", "country"], + {"city": ["Paris"], "country": ["France"]}, + ) + assert out[0] == "Location: Paris, France end" + + +def test_optional_block_gating_column_empty_is_dropped(): + # When the gating (first) column is empty the whole block is omitted; this + # behaviour is unchanged by the None coercion. + merged_prompt = "Location: [[{city}, {country}]] end" + out = _render( + merged_prompt, + ["city", "country"], + {"city": [""], "country": ["France"]}, + ) + assert out[0] == "Location: end" + + +def test_single_column_optional_block_gated_out_on_none(): + # Single-column blocks were already gated correctly (the sole column is the + # gate); confirm they stay unaffected. + merged_prompt = "Name: [[{name}]]!" + out = _render(merged_prompt, ["name"], {"name": [None, "Bob"]}) + assert out == ["Name: !", "Name: Bob!"] + + +def test_required_column_none_does_not_render_none(): + # A required (non-[[...]]) column that is None must not render as the + # literal "None" either; coercion happens at the row source, so both the + # required and optional branches are covered. + merged_prompt = "Location: {city}, {country} end" + out = _render( + merged_prompt, + ["city", "country"], + {"city": ["Paris"], "country": [None]}, + ) + assert out[0] == "Location: Paris, end" + assert "None" not in out[0] + + +def test_optional_block_falsy_but_present_gating_value_still_renders(): + # The gate keeps a block whenever the first column is not "". A falsy but + # real value (0) must not be treated as absent, so the block still renders. + merged_prompt = "Count: [[{n}]]!" + out = _render(merged_prompt, ["n"], {"n": [0]}) + assert out[0] == "Count: 0!" diff --git a/unsloth/chat_templates.py b/unsloth/chat_templates.py index 169b2dbd0e..dd1e433471 100644 --- a/unsloth/chat_templates.py +++ b/unsloth/chat_templates.py @@ -2185,7 +2185,16 @@ def _create_formatter(possible_columns, final_optional_prompts, user_column_name texts = [] for row_idx in range(n_rows): - row_values = {column: examples[column][row_idx] for column in columns} + # Coerce missing (None) columns to "" so they do not render as the + # literal string "None" in the emitted text. In a [[...]] block only + # the first column gates the block, so a later column can still be + # None here; required columns can be None too. Coercing at the source + # covers both; since None is now "", the gate below only needs to + # test for "" (an empty first column still drops the block). + row_values = { + column: ("" if (value := examples[column][row_idx]) is None else value) + for column in columns + } formatter_values = {} for formatter_template in formatter_templates: @@ -2196,7 +2205,7 @@ def _create_formatter(possible_columns, final_optional_prompts, user_column_name continue _, optional_name, prompt, needed_columns = formatter_template - if row_values[needed_columns[0]] not in (None, ""): + if row_values[needed_columns[0]] != "": prompt_values = {column: row_values[column] for column in needed_columns} formatter_values[optional_name] = prompt.format(**prompt_values) else: From 81f789ba85bb45c30fe7a8e60126112e07e3ddac Mon Sep 17 00:00:00 2001 From: ramisworld Date: Thu, 9 Jul 2026 09:32:51 +1200 Subject: [PATCH 007/372] Guard FP8 Triton launches with tensor device context (#6888) --------- Co-authored-by: oobabooga <112222186+oobabooga@users.noreply.github.com> --- tests/test_fp8_device_context.py | 249 +++++++++++++++++++++++++++++++ unsloth/kernels/fp8.py | 87 ++++++----- 2 files changed, 300 insertions(+), 36 deletions(-) create mode 100644 tests/test_fp8_device_context.py diff --git a/tests/test_fp8_device_context.py b/tests/test_fp8_device_context.py new file mode 100644 index 0000000000..2eea35f4e6 --- /dev/null +++ b/tests/test_fp8_device_context.py @@ -0,0 +1,249 @@ +from __future__ import annotations + +import ast +from contextlib import nullcontext +from pathlib import Path +from types import SimpleNamespace + +import pytest + + +REPO_ROOT = Path(__file__).resolve().parents[1] +FP8_SOURCE = REPO_ROOT / "unsloth" / "kernels" / "fp8.py" + + +class _FakeDeviceModule: + def __init__(self, device_count: int) -> None: + self._device_count = device_count + self.device_calls = [] + + def device_count(self) -> int: + return self._device_count + + def device(self, device): + self.device_calls.append(device) + return ("device-context", device) + + +class _FakeTorch: + Tensor = object + + def __init__( + self, + cuda_device_count: int, + xpu_device_count: int = 0, + ) -> None: + self.cuda = _FakeDeviceModule(cuda_device_count) + self.xpu = _FakeDeviceModule(xpu_device_count) + + +class _LaunchVisitor(ast.NodeVisitor): + def __init__(self) -> None: + self.guarded_launches: set[str] = set() + self.unguarded_launches: set[str] = set() + self._inside_fp8_device_context = 0 + + def visit_With(self, node: ast.With) -> None: + enters_context = any( + isinstance(item.context_expr, ast.Call) + and isinstance(item.context_expr.func, ast.Name) + and item.context_expr.func.id == "_fp8_triton_device_context" + for item in node.items + ) + if enters_context: + self._inside_fp8_device_context += 1 + for statement in node.body: + self.visit(statement) + if enters_context: + self._inside_fp8_device_context -= 1 + + def visit_Call(self, node: ast.Call) -> None: + launch_name = self._triton_launch_name(node) + if launch_name is not None: + if self._inside_fp8_device_context: + self.guarded_launches.add(launch_name) + else: + self.unguarded_launches.add(launch_name) + self.generic_visit(node) + + @staticmethod + def _triton_launch_name(node: ast.Call) -> str | None: + if isinstance(node.func, ast.Name) and node.func.id == "triton_quantize_fp8_block": + return node.func.id + if not isinstance(node.func, ast.Subscript): + return None + if not isinstance(node.func.value, ast.Name): + return None + return node.func.value.id + + +def _load_device_context_helper(fake_torch: _FakeTorch): + source = FP8_SOURCE.read_text() + tree = ast.parse(source) + for node in tree.body: + if isinstance(node, ast.FunctionDef) and node.name == "_fp8_triton_device_context": + namespace = {"torch": fake_torch, "nullcontext": nullcontext} + exec(ast.get_source_segment(source, node), namespace) + return namespace["_fp8_triton_device_context"] + raise AssertionError("_fp8_triton_device_context was not found") + + +def test_fp8_device_context_selects_cuda_tensor_device_on_multi_gpu() -> None: + fake_torch = _FakeTorch(cuda_device_count = 2) + helper = _load_device_context_helper(fake_torch) + tensor = SimpleNamespace(device = SimpleNamespace(type = "cuda")) + + context = helper(tensor) + + assert context == ("device-context", tensor.device) + assert fake_torch.cuda.device_calls == [tensor.device] + + +def test_fp8_device_context_is_noop_for_single_cuda_device() -> None: + fake_torch = _FakeTorch(cuda_device_count = 1) + helper = _load_device_context_helper(fake_torch) + tensor = SimpleNamespace(device = SimpleNamespace(type = "cuda")) + + context = helper(tensor) + + assert isinstance(context, nullcontext) + assert fake_torch.cuda.device_calls == [] + + +def test_fp8_device_context_selects_xpu_tensor_device_on_multi_gpu() -> None: + fake_torch = _FakeTorch(cuda_device_count = 0, xpu_device_count = 2) + helper = _load_device_context_helper(fake_torch) + tensor = SimpleNamespace(device = SimpleNamespace(type = "xpu")) + + context = helper(tensor) + + assert context == ("device-context", tensor.device) + assert fake_torch.xpu.device_calls == [tensor.device] + + +def test_fp8_device_context_is_noop_for_single_xpu_device() -> None: + fake_torch = _FakeTorch(cuda_device_count = 0, xpu_device_count = 1) + helper = _load_device_context_helper(fake_torch) + tensor = SimpleNamespace(device = SimpleNamespace(type = "xpu")) + + context = helper(tensor) + + assert isinstance(context, nullcontext) + assert fake_torch.xpu.device_calls == [] + + +def test_fp8_device_context_is_noop_for_non_cuda_tensor() -> None: + fake_torch = _FakeTorch(cuda_device_count = 8) + helper = _load_device_context_helper(fake_torch) + tensor = SimpleNamespace(device = SimpleNamespace(type = "cpu")) + + context = helper(tensor) + + assert isinstance(context, nullcontext) + assert fake_torch.cuda.device_calls == [] + + +def test_fp8_triton_launches_enter_tensor_device_context() -> None: + tree = ast.parse(FP8_SOURCE.read_text()) + function_names = {node.name for node in ast.walk(tree) if isinstance(node, ast.FunctionDef)} + assert "_fp8_triton_device_context" in function_names + + visitor = _LaunchVisitor() + visitor.visit(tree) + + expected_launches = { + "weight_dequant_kernel", + "act_quant_kernel", + "_w8a8_block_fp8_matmul", + "triton_quantize_fp8_block", + } + assert expected_launches <= visitor.guarded_launches + assert not (expected_launches & visitor.unguarded_launches) + + +def _require_two_cuda_devices(): + torch = pytest.importorskip("torch") + pytest.importorskip("triton") + + if not torch.cuda.is_available() or torch.cuda.device_count() < 2: + pytest.skip("requires at least two CUDA devices") + return torch + + +def test_weight_dequant_block_runs_on_tensor_device_when_current_device_differs() -> None: + torch = _require_two_cuda_devices() + from unsloth.kernels.fp8 import weight_dequant_block + + previous_device = torch.cuda.current_device() + try: + torch.cuda.set_device(0) + x = torch.arange(256 * 256, device = "cuda:1", dtype = torch.float32).reshape(256, 256) + scales = torch.tensor([[1.0, 2.0], [3.0, 4.0]], device = "cuda:1", dtype = torch.float32) + + actual = weight_dequant_block(x, scales, block_size = 128, dtype = torch.float32) + + expanded_scales = scales.repeat_interleave(128, dim = 0).repeat_interleave(128, dim = 1) + expected = x * expanded_scales + + assert actual.device == x.device + assert torch.cuda.current_device() == 0 + torch.testing.assert_close(actual, expected) + finally: + torch.cuda.set_device(previous_device) + + +def test_act_quant_runs_on_tensor_device_when_current_device_differs() -> None: + torch = _require_two_cuda_devices() + if not hasattr(torch, "float8_e4m3fn"): + pytest.skip("requires torch.float8_e4m3fn") + if torch.cuda.get_device_capability(1)[0] < 9: + pytest.skip("requires FP8-capable CUDA hardware") + + from unsloth.kernels.fp8 import act_quant + + previous_device = torch.cuda.current_device() + try: + torch.cuda.set_device(0) + x = torch.arange(256, device = "cuda:1", dtype = torch.float32).reshape(2, 128) + + y, scales = act_quant(x, block_size = 128) + + assert y.device == x.device + assert scales.device == x.device + assert torch.cuda.current_device() == 0 + finally: + torch.cuda.set_device(previous_device) + + +def test_w8a8_block_fp8_matmul_triton_runs_on_tensor_device_when_current_device_differs() -> None: + torch = _require_two_cuda_devices() + if not hasattr(torch, "float8_e4m3fn"): + pytest.skip("requires torch.float8_e4m3fn") + if torch.cuda.get_device_capability(1)[0] < 9: + pytest.skip("requires FP8-capable CUDA hardware") + + from unsloth.kernels.fp8 import w8a8_block_fp8_matmul_triton + + previous_device = torch.cuda.current_device() + try: + torch.cuda.set_device(0) + A = torch.ones((128, 128), device = "cuda:1", dtype = torch.float32).to(torch.float8_e4m3fn) + B = torch.ones((128, 128), device = "cuda:1", dtype = torch.float32).to(torch.float8_e4m3fn) + As = torch.ones((128, 1), device = "cuda:1", dtype = torch.float32) + Bs = torch.ones((1, 1), device = "cuda:1", dtype = torch.float32) + + actual = w8a8_block_fp8_matmul_triton( + A, + B, + As, + Bs, + block_size = [128, 128], + output_dtype = torch.float32, + ) + + expected = torch.full((128, 128), 128.0, device = "cuda:1", dtype = torch.float32) + assert actual.device == A.device + assert torch.cuda.current_device() == 0 + torch.testing.assert_close(actual, expected) + finally: + torch.cuda.set_device(previous_device) diff --git a/unsloth/kernels/fp8.py b/unsloth/kernels/fp8.py index 935ffbb447..4efc4bd5d3 100644 --- a/unsloth/kernels/fp8.py +++ b/unsloth/kernels/fp8.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import os +from contextlib import nullcontext import torch import torch.nn as nn import triton @@ -24,6 +25,15 @@ from unsloth_zoo.temporary_patches.common import torch_compile torch_matmul = torch.matmul + +def _fp8_triton_device_context(tensor: torch.Tensor): + if tensor.device.type == "cuda" and torch.cuda.device_count() > 1: + return torch.cuda.device(tensor.device) + if tensor.device.type == "xpu" and hasattr(torch, "xpu") and torch.xpu.device_count() > 1: + return torch.xpu.device(tensor.device) + return nullcontext() + + try: from transformers.integrations.finegrained_fp8 import FP8Linear except: @@ -95,7 +105,8 @@ def weight_dequant_block( triton.cdiv(M, meta["BLOCK_SIZE"]), triton.cdiv(N, meta["BLOCK_SIZE"]), ) - weight_dequant_kernel[grid](x, s, y, M, N, BLOCK_SIZE = block_size) + with _fp8_triton_device_context(x): + weight_dequant_kernel[grid](x, s, y, M, N, BLOCK_SIZE = block_size) return y @@ -149,7 +160,8 @@ def act_quant(x: torch.Tensor, block_size: int = 128) -> tuple[torch.Tensor, tor def grid(meta): return (triton.cdiv(x.numel(), meta["BLOCK_SIZE"]),) - act_quant_kernel[grid](x, y, s, BLOCK_SIZE = block_size) + with _fp8_triton_device_context(x): + act_quant_kernel[grid](x, y, s, BLOCK_SIZE = block_size) return y, s @@ -274,32 +286,33 @@ def w8a8_block_fp8_matmul_triton( def grid(META): return (triton.cdiv(M, META["BLOCK_SIZE_M"]) * triton.cdiv(N, META["BLOCK_SIZE_N"]),) - _w8a8_block_fp8_matmul[grid]( - A, - B, - C, - As, - Bs, - M, - N, - K, - block_n, - block_k, - A.stride(-2), - A.stride(-1), - B.stride(1), - B.stride(0), - C.stride(-2), - C.stride(-1), - As.stride(-2), - As.stride(-1), - Bs.stride(1), - Bs.stride(0), - BLOCK_SIZE_M = BLOCK_SIZE_M, - BLOCK_SIZE_N = BLOCK_SIZE_N, - BLOCK_SIZE_K = BLOCK_SIZE_K, - GROUP_SIZE_M = 8, - ) + with _fp8_triton_device_context(A): + _w8a8_block_fp8_matmul[grid]( + A, + B, + C, + As, + Bs, + M, + N, + K, + block_n, + block_k, + A.stride(-2), + A.stride(-1), + B.stride(1), + B.stride(0), + C.stride(-2), + C.stride(-1), + As.stride(-2), + As.stride(-1), + Bs.stride(1), + Bs.stride(0), + BLOCK_SIZE_M = BLOCK_SIZE_M, + BLOCK_SIZE_N = BLOCK_SIZE_N, + BLOCK_SIZE_K = BLOCK_SIZE_K, + GROUP_SIZE_M = 8, + ) return C @@ -311,13 +324,14 @@ def torchao_block_matmul( block_size: tuple[int, int], output_dtype: torch.dtype = torch.bfloat16, ): - out = torchao_blockwise_gemm( - act_q.contiguous(), - act_scale.contiguous(), - weight_q.contiguous(), - weight_scale.contiguous(), - block_size = block_size[1], - ) + with _fp8_triton_device_context(act_q): + out = torchao_blockwise_gemm( + act_q.contiguous(), + act_scale.contiguous(), + weight_q.contiguous(), + weight_scale.contiguous(), + block_size = block_size[1], + ) return out.to(output_dtype) @@ -540,7 +554,8 @@ class FP8_fbgemm_block_linear(torch.autograd.Function): f"Weight shape {weight.shape} and scales shape {weight_scale.shape} is not compatible with block size {bs_n, bs_k}" ) - xq, xs = triton_quantize_fp8_block(X, bs_m, bs_n, None) + with _fp8_triton_device_context(X): + xq, xs = triton_quantize_fp8_block(X, bs_m, bs_n, None) # TODO: WARNING - diverges from baseline for high X values, producing # gibberish / high starting loss. Do not use until resolved; kept for a # future headstart. From 3b73cd88293728b3ba173e235d405cec40f7a42b Mon Sep 17 00:00:00 2001 From: Nilay <118994073+NilayYadav@users.noreply.github.com> Date: Thu, 9 Jul 2026 08:33:03 +0530 Subject: [PATCH 008/372] Fix per-block ID collisions and add block cleanup for unstructured uploads (#6944) * unstructured block removal * Enhance unstructured block handling * Restrict block cleanup to upload UIDs * cleanup for seed block uploads * upload cleanup queue for unstructured blocks in recipe studio * Fix unstructured upload cleanup edge cases * Fix unstructured upload import ownership * Fix-unstructured-import-path-ownership * Guard failed-delete restore against stale block in unstructured drop zone * Drain queued upload cleanups when autosave is skipped --------- Co-authored-by: Lee Jackson <130007945+Imagineer99@users.noreply.github.com> Co-authored-by: imagineer99 Co-authored-by: Daniel Han --- studio/backend/routes/data_recipe/seed.py | 37 ++++++ studio/backend/tests/test_data_recipe_seed.py | 97 +++++++++++++++ .../src/features/recipe-studio/api/index.ts | 10 ++ .../dialogs/seed/seed-dialog.tsx | 69 ++++++++++- .../dialogs/seed/unstructured-drop-zone.tsx | 37 ++++-- .../hooks/use-recipe-persistence.ts | 86 ++++++++++++-- .../recipe-studio/stores/recipe-studio.ts | 61 +++++++++- .../src/features/recipe-studio/types/index.ts | 2 + .../recipe-studio/utils/config-factories.ts | 43 +++++++ .../recipe-studio/utils/import/importer.ts | 112 +++++++++++------- .../import/parsers/seed-config-parser.ts | 14 ++- .../utils/payload/build-payload.ts | 3 + .../recipe-studio/utils/payload/types.ts | 2 + 13 files changed, 502 insertions(+), 71 deletions(-) diff --git a/studio/backend/routes/data_recipe/seed.py b/studio/backend/routes/data_recipe/seed.py index 57a291291e..a5b75b7335 100644 --- a/studio/backend/routes/data_recipe/seed.py +++ b/studio/backend/routes/data_recipe/seed.py @@ -10,6 +10,7 @@ import binascii import json import os import re +import shutil from itertools import islice from pathlib import Path from typing import Any @@ -59,6 +60,9 @@ UNSTRUCTURED_ALLOWED_EXTS = {".pdf", ".docx", ".txt", ".md"} SEED_UPLOAD_DIR = seed_uploads_root() UNSTRUCTURED_UPLOAD_ROOT = unstructured_uploads_root() _SAFE_ID_RE = re.compile(r"^[a-zA-Z0-9_-]+$") +# Frontend-generated upload namespace (UUID4 hex). Legacy node ids (n1, ...) +# never match: those directories can be shared by several recipes. +_UPLOAD_UID_RE = re.compile(r"^[0-9a-f]{32}$") def _validate_safe_id(value: str, label: str) -> str: @@ -580,6 +584,39 @@ async def remove_unstructured_file(block_id: str, file_id: str): return {"status": "ok"} +@router.delete("/seed/unstructured-block/{block_id}") +async def remove_unstructured_block(block_id: str): + """Delete a block's upload directory; files on disk still count toward its quota. + + Only uid-namespaced directories may be bulk-deleted: they have exactly one + owning block. Legacy node-id directories (n1, ...) can be shared by other + recipes, so they are managed file-by-file instead. + """ + _validate_safe_id(block_id, "block_id") + if not _UPLOAD_UID_RE.match(block_id): + raise HTTPException(400, "Invalid block_id: only uid-namespaced blocks can be deleted") + + block_dir = (UNSTRUCTURED_UPLOAD_ROOT / block_id).resolve() + if not block_dir.is_relative_to(UNSTRUCTURED_UPLOAD_ROOT.resolve()): + raise HTTPException(400, "Invalid block_id: outside upload root") + if not block_dir.exists(): + return {"status": "ok", "deleted": False} + + try: + shutil.rmtree(block_dir) + except OSError as exc: + raise log_and_http_error( + exc, + 500, + "failed to delete uploaded files", + event = "data_recipe.seed.unstructured_block_delete_failed", + log = logger, + ) from exc + if block_dir.exists(): + raise HTTPException(500, "failed to delete uploaded files") + return {"status": "ok", "deleted": True} + + @router.post("/seed/inspect-upload", response_model = SeedInspectResponse) def inspect_seed_upload(payload: SeedInspectUploadRequest) -> SeedInspectResponse: if payload.file_ids is not None: diff --git a/studio/backend/tests/test_data_recipe_seed.py b/studio/backend/tests/test_data_recipe_seed.py index 09e22116ed..58bbd24061 100644 --- a/studio/backend/tests/test_data_recipe_seed.py +++ b/studio/backend/tests/test_data_recipe_seed.py @@ -124,3 +124,100 @@ def test_unstructured_upload_import_errors_stay_generic(monkeypatch, tmp_path, e assert result.status == "error" assert result.error == "Text extraction failed." assert _block_files(seed_route) == [] + + +_TEST_UPLOAD_UID = "0f" * 16 + + +def test_remove_unstructured_block_deletes_directory(monkeypatch, tmp_path): + seed_route = _load_seed_route(monkeypatch, tmp_path) + _run_upload(seed_route, "notes.txt", b"hello", block_id = _TEST_UPLOAD_UID) + assert _block_files(seed_route, _TEST_UPLOAD_UID) != [] + + result = asyncio.run(seed_route.remove_unstructured_block(_TEST_UPLOAD_UID)) + + assert result == {"status": "ok", "deleted": True} + assert not (seed_route.UNSTRUCTURED_UPLOAD_ROOT / _TEST_UPLOAD_UID).exists() + + +def test_remove_unstructured_block_missing_directory_is_ok(monkeypatch, tmp_path): + seed_route = _load_seed_route(monkeypatch, tmp_path) + + result = asyncio.run(seed_route.remove_unstructured_block(_TEST_UPLOAD_UID)) + + assert result == {"status": "ok", "deleted": False} + + +def test_remove_unstructured_block_rejects_unsafe_ids(monkeypatch, tmp_path): + seed_route = _load_seed_route(monkeypatch, tmp_path) + + with pytest.raises(seed_route.HTTPException) as exc: + asyncio.run(seed_route.remove_unstructured_block("../escape")) + + assert exc.value.status_code == 400 + + +def test_remove_unstructured_block_rejects_legacy_node_ids(monkeypatch, tmp_path): + seed_route = _load_seed_route(monkeypatch, tmp_path) + _run_upload(seed_route, "notes.txt", b"hello", block_id = "n1") + assert _block_files(seed_route, "n1") != [] + + with pytest.raises(seed_route.HTTPException) as exc: + asyncio.run(seed_route.remove_unstructured_block("n1")) + + assert exc.value.status_code == 400 + assert _block_files(seed_route, "n1") != [] + + +def test_remove_unstructured_block_rejects_symlink_escape(monkeypatch, tmp_path): + seed_route = _load_seed_route(monkeypatch, tmp_path) + outside = tmp_path / "outside" + outside.mkdir() + (outside / "victim.txt").write_text("keep me") + root = seed_route.UNSTRUCTURED_UPLOAD_ROOT + root.mkdir(parents = True) + (root / _TEST_UPLOAD_UID).symlink_to(outside) + + with pytest.raises(seed_route.HTTPException) as exc: + asyncio.run(seed_route.remove_unstructured_block(_TEST_UPLOAD_UID)) + + assert exc.value.status_code == 400 + assert (outside / "victim.txt").exists() + + +def test_remove_unstructured_block_fails_if_directory_remains(monkeypatch, tmp_path): + seed_route = _load_seed_route(monkeypatch, tmp_path) + root = seed_route.UNSTRUCTURED_UPLOAD_ROOT + block_dir = root / _TEST_UPLOAD_UID + block_dir.mkdir(parents = True) + (block_dir / "victim.txt").write_text("keep me") + + calls = [] + + def noop_rmtree(path, *args, **kwargs): + calls.append((path, args, kwargs)) + + monkeypatch.setattr(seed_route.shutil, "rmtree", noop_rmtree) + + with pytest.raises(seed_route.HTTPException) as exc: + asyncio.run(seed_route.remove_unstructured_block(_TEST_UPLOAD_UID)) + + assert calls + assert exc.value.status_code == 500 + assert block_dir.exists() + + +def test_total_upload_quota_is_scoped_per_block(monkeypatch, tmp_path): + seed_route = _load_seed_route(monkeypatch, tmp_path) + monkeypatch.setattr(seed_route, "UNSTRUCTURED_RECIPE_UPLOAD_TOTAL_MAX_BYTES", 10) + + first = _run_upload(seed_route, "a.txt", b"123456789") + assert first.status == "ok" + + with pytest.raises(seed_route.HTTPException) as exc: + _run_upload(seed_route, "b.txt", b"123") + assert exc.value.status_code == 413 + + # Another block starts with its own untouched budget. + other = _run_upload(seed_route, "c.txt", b"123", block_id = "other") + assert other.status == "ok" diff --git a/studio/frontend/src/features/recipe-studio/api/index.ts b/studio/frontend/src/features/recipe-studio/api/index.ts index 4fe24b1f6d..f4e8167cb3 100644 --- a/studio/frontend/src/features/recipe-studio/api/index.ts +++ b/studio/frontend/src/features/recipe-studio/api/index.ts @@ -494,3 +494,13 @@ export async function removeUnstructuredFile( throw new Error("Failed to remove file"); } } + +export async function removeUnstructuredBlock(blockId: string): Promise { + const res = await authFetch( + `${DATA_DESIGNER_API_BASE}/seed/unstructured-block/${encodeURIComponent(blockId)}`, + { method: "DELETE" }, + ); + if (!res.ok && res.status !== 404) { + throw new Error("Failed to remove uploaded files"); + } +} diff --git a/studio/frontend/src/features/recipe-studio/dialogs/seed/seed-dialog.tsx b/studio/frontend/src/features/recipe-studio/dialogs/seed/seed-dialog.tsx index 0ed7eeed75..53f8566090 100644 --- a/studio/frontend/src/features/recipe-studio/dialogs/seed/seed-dialog.tsx +++ b/studio/frontend/src/features/recipe-studio/dialogs/seed/seed-dialog.tsx @@ -53,6 +53,11 @@ import { inspectSeedDataset, inspectSeedUpload, } from "../../api"; +import { useRecipeStudioStore } from "../../stores/recipe-studio"; +import { + makeUnstructuredUploadUid, + resolveUnstructuredUploadBlockId, +} from "../../utils/config-factories"; import { resolveImagePreview } from "../../utils/image-preview"; import type { GithubItemType, @@ -597,6 +602,41 @@ export function SeedDialog({ const mode = config.seed_source_type ?? "hf"; const previewEmpty = getPreviewEmptyStateCopy(mode); + const queueUploadCleanup = useRecipeStudioStore( + (state) => state.queueUploadCleanup, + ); + + // config.id collides across recipes (ids reset to n1 on import); use a + // stable per-block uid instead. Generate one synchronously so the first + // rendered drop zone cannot upload under a legacy node id. + const uploadUid = config.unstructured_upload_uid?.trim() ?? ""; + const unstructuredFileCount = config.unstructured_file_ids?.length ?? 0; + const generatedUploadUidRef = useRef(null); + if ( + mode === "unstructured" && + !uploadUid && + unstructuredFileCount === 0 && + generatedUploadUidRef.current === null + ) { + generatedUploadUidRef.current = makeUnstructuredUploadUid(); + } + const uploadBlockId = resolveUnstructuredUploadBlockId({ + configId: config.id, + uploadUid, + generatedUploadUid: generatedUploadUidRef.current, + unstructuredFileCount, + }); + + useEffect(() => { + if (mode !== "unstructured") return; + if (uploadUid) return; + if (unstructuredFileCount > 0) return; + const nextUid = + generatedUploadUidRef.current ?? makeUnstructuredUploadUid(); + generatedUploadUidRef.current = nextUid; + onUpdate({ unstructured_upload_uid: nextUid }); + }, [mode, uploadUid, unstructuredFileCount, onUpdate]); + const prevModeRef = useRef(mode); useEffect(() => { const prevMode = prevModeRef.current; @@ -720,6 +760,11 @@ export function SeedDialog({ subset: config.hf_subset?.trim() || undefined, preview_size: 10, }); + // Queue the block's upload directory for deletion after the next + // save; only uid-namespaced directories qualify (single owner). + if (uploadUid && unstructuredFileCount > 0) { + queueUploadCleanup(uploadUid); + } onUpdate({ hf_path: response.resolved_path, seed_columns: response.columns, @@ -730,6 +775,7 @@ export function SeedDialog({ hf_split: response.split ?? "", hf_subset: response.subset ?? "", local_file_name: "", + unstructured_upload_uid: "", unstructured_file_ids: [], unstructured_file_names: [], unstructured_file_sizes: [], @@ -754,6 +800,11 @@ export function SeedDialog({ content_base64: payload, preview_size: 10, }); + // Queue the block's upload directory for deletion after the next + // save; only uid-namespaced directories qualify (single owner). + if (uploadUid && unstructuredFileCount > 0) { + queueUploadCleanup(uploadUid); + } onUpdate({ hf_path: response.resolved_path, seed_columns: response.columns, @@ -765,6 +816,7 @@ export function SeedDialog({ hf_subset: "", hf_split: "", local_file_name: localFile.name, + unstructured_upload_uid: "", unstructured_file_ids: [], unstructured_file_names: [], unstructured_file_sizes: [], @@ -789,7 +841,7 @@ export function SeedDialog({ const { chunkSize, chunkOverlap } = resolveChunking(config); const response = await inspectSeedUpload({ - block_id: config.id, + block_id: uploadBlockId, file_ids: fileIds, file_names: fileNames, preview_size: 10, @@ -827,7 +879,18 @@ export function SeedDialog({ setIsInspecting(false); } }, - [config, getCurrentLoadKey, localFile, mode, onUpdate, unstructuredFiles], + [ + config, + getCurrentLoadKey, + localFile, + mode, + onUpdate, + queueUploadCleanup, + unstructuredFiles, + unstructuredFileCount, + uploadBlockId, + uploadUid, + ], ); useEffect(() => { @@ -997,7 +1060,7 @@ export function SeedDialog({ {mode === "unstructured" && ( (null); const filesRef = useRef(files); + const blockIdRef = useRef(blockId); + const mountedRef = useRef(true); const [isDragOver, setIsDragOver] = useState(false); useEffect(() => { filesRef.current = files; - }, [files]); + blockIdRef.current = blockId; + }, [files, blockId]); + useEffect(() => () => { + mountedRef.current = false; + }, []); const totalSize = files.reduce((sum, f) => sum + f.size, 0); @@ -134,15 +140,32 @@ export function UnstructuredDropZone({ if (entry.status === "uploading" && entry.abortController) { entry.abortController.abort(); } - if ( + const needsServerRemove = entry.id && entry.status === "ok" && - !deletedIdsRef.current.has(entry.id) - ) { - deletedIdsRef.current.add(entry.id); - void removeUnstructuredFile(blockId, entry.id).catch(() => {}); - } + !deletedIdsRef.current.has(entry.id); onFilesChange((prev) => prev.filter((_, i) => i !== index)); + if (!needsServerRemove) return; + deletedIdsRef.current.add(entry.id); + removeUnstructuredFile(blockId, entry.id).catch(() => { + // Skip if the drop zone unmounted or its block changed: the id no + // longer belongs here and restoring would leak it into another block. + if (!mountedRef.current || blockIdRef.current !== blockId) return; + // Still exists server-side (counts toward quota); restore it at its + // original position. + deletedIdsRef.current.delete(entry.id); + onFilesChange((prev) => { + const next = [...prev]; + next.splice(Math.min(index, next.length), 0, { + id: entry.id, + name: entry.name, + size: entry.size, + status: "ok", + error: "Remove failed — try again", + }); + return next; + }); + }); }, [blockId, onFilesChange], ); diff --git a/studio/frontend/src/features/recipe-studio/hooks/use-recipe-persistence.ts b/studio/frontend/src/features/recipe-studio/hooks/use-recipe-persistence.ts index 0417a6dd22..2d91272469 100644 --- a/studio/frontend/src/features/recipe-studio/hooks/use-recipe-persistence.ts +++ b/studio/frontend/src/features/recipe-studio/hooks/use-recipe-persistence.ts @@ -4,11 +4,13 @@ import { useCallback, useEffect, useMemo, useState } from "react"; import { toastError, toastSuccess } from "@/shared/toast"; import { normalizeNonEmptyName } from "@/utils"; +import { removeUnstructuredBlock } from "../api"; import { buildSignature, copyTextToClipboard, formatSavedLabel, } from "../executions/execution-helpers"; +import { useRecipeStudioStore } from "../stores/recipe-studio"; import { importRecipePayload, type RecipeSnapshot } from "../utils/import"; import type { RecipePayloadResult } from "../utils/payload/types"; @@ -72,7 +74,10 @@ function stripApiKeys(value: unknown): unknown { !Array.isArray(output.env) ) { output.env = Object.fromEntries( - Object.keys(output.env as Record).map((envKey) => [envKey, ""]), + Object.keys(output.env as Record).map((envKey) => [ + envKey, + "", + ]), ); } return output; @@ -82,10 +87,7 @@ function inferHfRepoIdFromPath(pathValue: unknown): string { if (typeof pathValue !== "string") { return ""; } - const parts = pathValue - .trim() - .split("/") - .filter(Boolean); + const parts = pathValue.trim().split("/").filter(Boolean); if (parts.length >= 3 && parts[0] === "datasets") { return `${parts[1]}/${parts[2]}`; } @@ -126,8 +128,7 @@ function sanitizeSeedForShare(payload: unknown): unknown { typeof ui?.seed_source_type === "string" ? ui.seed_source_type : null; const sourceType = typeof source?.seed_type === "string" ? source.seed_type : null; - const shouldResetHfState = - sourceType === "hf" || uiSourceType === "hf"; + const shouldResetHfState = sourceType === "hf" || uiSourceType === "hf"; const shouldResetLocalState = sourceType === "local" || sourceType === "unstructured" || @@ -144,6 +145,7 @@ function sanitizeSeedForShare(payload: unknown): unknown { ui.seed_drop_columns = []; ui.seed_preview_rows = []; ui.local_file_name = ""; + ui.unstructured_upload_uid = ""; ui.unstructured_file_ids = []; ui.unstructured_file_names = []; ui.unstructured_file_sizes = []; @@ -165,6 +167,7 @@ function sanitizeSeedForShare(payload: unknown): unknown { ui.seed_drop_columns = []; ui.seed_preview_rows = []; ui.local_file_name = ""; + ui.unstructured_upload_uid = ""; ui.unstructured_file_ids = []; ui.unstructured_file_names = []; ui.unstructured_file_sizes = []; @@ -174,6 +177,43 @@ function sanitizeSeedForShare(payload: unknown): unknown { return root; } +// Delete queued upload directories once a save stops referencing them, so a +// reload before autosave can never leave the saved recipe pointing at +// already-deleted files. Skips any uid the just-saved payload still uses. +function drainQueuedUploadCleanups( + savedPayload: RecipePayloadResult["payload"], +): void { + const pending = useRecipeStudioStore.getState().pendingUploadCleanups; + if (pending.length === 0) { + return; + } + const ui = + savedPayload && typeof savedPayload === "object" + ? (savedPayload as { ui?: Record }).ui + : undefined; + const savedUid = + ui && typeof ui.unstructured_upload_uid === "string" + ? ui.unstructured_upload_uid + : ""; + const ready = pending.filter((uid) => uid !== savedUid); + if (ready.length === 0) { + return; + } + for (const uid of ready) { + void removeUnstructuredBlock(uid) + .then(() => { + useRecipeStudioStore.setState((state) => ({ + pendingUploadCleanups: state.pendingUploadCleanups.filter( + (pendingUid) => pendingUid !== uid, + ), + })); + }) + .catch((error) => { + console.warn("Failed to clean up uploaded documents:", error); + }); + } +} + export function useRecipePersistence({ recipeId, initialRecipeName, @@ -202,8 +242,10 @@ export function useRecipePersistence({ () => buildSignature(normalizedWorkflowName, currentPayload), [currentPayload, normalizedWorkflowName], ); - const isDirty = savedSignature.length > 0 && currentSignature !== savedSignature; - const saveTone: SaveTone = !isDirty && Boolean(lastSavedAt) ? "success" : "error"; + const isDirty = + savedSignature.length > 0 && currentSignature !== savedSignature; + const saveTone: SaveTone = + !isDirty && Boolean(lastSavedAt) ? "success" : "error"; const savedAtLabel = formatSavedLabel(lastSavedAt); useEffect(() => { @@ -214,7 +256,9 @@ export function useRecipePersistence({ setLastSavedAt(initialSavedAt); setCopied(false); - const parsed = importRecipePayload(JSON.stringify(initialPayload)); + const parsed = importRecipePayload(JSON.stringify(initialPayload), { + preserveUnstructuredUploads: true, + }); if (parsed.snapshot) { loadRecipe(parsed.snapshot); } else { @@ -252,6 +296,7 @@ export function useRecipePersistence({ }); setLastSavedAt(result.updatedAt); setSavedSignature(buildSignature(nextName, currentPayload)); + drainQueuedUploadCleanups(currentPayload); } catch (error) { console.error("Save recipe failed:", error); toastError("Save failed", "Could not save recipe."); @@ -270,11 +315,28 @@ export function useRecipePersistence({ return () => window.clearTimeout(timeoutId); }, [isDirty, persistRecipe, saveLoading]); + // Drain queued cleanups even when autosave is skipped: a net-zero edit (add + // then remove an unstructured seed before the 800ms debounce) keeps isDirty + // false, so the autosave effect never drains and the queued uid leaks its + // upload dir. Not-dirty means currentPayload equals the saved recipe, and + // drain skips the uid it still references, so only dirs no saved recipe + // points at are deleted (keeps the save-first invariant). + useEffect(() => { + if (!initialRecipeReady || isDirty || saveLoading) { + return; + } + drainQueuedUploadCleanups(currentPayload); + }, [currentPayload, initialRecipeReady, isDirty, saveLoading]); + const copyRecipe = useCallback(async (): Promise => { setCopied(false); try { - const safePayload = sanitizeSeedForShare(stripApiKeys(payloadResult.payload)); - const ok = await copyTextToClipboard(JSON.stringify(safePayload, null, 2)); + const safePayload = sanitizeSeedForShare( + stripApiKeys(payloadResult.payload), + ); + const ok = await copyTextToClipboard( + JSON.stringify(safePayload, null, 2), + ); if (!ok) { throw new Error("Clipboard not available."); } diff --git a/studio/frontend/src/features/recipe-studio/stores/recipe-studio.ts b/studio/frontend/src/features/recipe-studio/stores/recipe-studio.ts index 8659cad4fb..a1ff72ee9b 100644 --- a/studio/frontend/src/features/recipe-studio/stores/recipe-studio.ts +++ b/studio/frontend/src/features/recipe-studio/stores/recipe-studio.ts @@ -36,6 +36,7 @@ import { } from "../utils/handles"; import type { RecipeSnapshot } from "../utils/import"; import { getLayoutedElements } from "../utils/layout"; +import { makeUnstructuredUploadUid } from "../utils/config-factories"; import { centerModelInfraNodes, optimizeModelInfraEdgeHandles, @@ -76,6 +77,12 @@ type RecipeStudioState = { nextId: number; nextY: number; fitViewTick: number; + // Upload-uid directories whose owning block dropped them; server-side + // deletion is deferred until a save no longer references them, so a + // reload before autosave cannot leave a saved recipe pointing at + // deleted files. + pendingUploadCleanups: string[]; + queueUploadCleanup: (uid: string) => void; setSheetOpen: (open: boolean) => void; setSheetView: (view: SheetView) => void; setProcessors: (processors: RecipeProcessorConfig[]) => void; @@ -137,6 +144,7 @@ const INITIAL_STATE = { nextId: 3, nextY: 280, fitViewTick: 0, + pendingUploadCleanups: [], } satisfies Pick< RecipeStudioState, | "nodes" @@ -154,6 +162,7 @@ const INITIAL_STATE = { | "nextId" | "nextY" | "fitViewTick" + | "pendingUploadCleanups" >; function buildAddedNodeState( @@ -269,6 +278,20 @@ function isModelSemanticEdge( ); } +// Upload uid of a seed block whose server-side directory becomes orphaned +// when the block drops it. Only uid directories qualify (single owner); +// legacy node-id directories can be shared by other recipes. +function seedUploadCleanupUid(config: NodeConfig | undefined): string | null { + if (!config || config.kind !== "seed") { + return null; + } + const uid = config.unstructured_upload_uid?.trim(); + if (!uid || !config.unstructured_file_ids?.length) { + return null; + } + return uid; +} + export const useRecipeStudioStore = create((set, get) => ({ ...INITIAL_STATE, setSheetOpen: (open) => set({ sheetOpen: open }), @@ -278,6 +301,12 @@ export const useRecipeStudioStore = create((set, get) => ({ setDialogOpen: (open) => set({ dialogOpen: open }), setExecutionLocked: (locked) => set({ executionLocked: locked }), resetRecipe: () => set(INITIAL_STATE), + queueUploadCleanup: (uid) => + set((state) => + state.pendingUploadCleanups.includes(uid) + ? state + : { pendingUploadCleanups: [...state.pendingUploadCleanups, uid] }, + ), selectConfig: (id) => set({ activeConfigId: id, dialogOpen: false }), openConfig: (id) => set({ activeConfigId: id, dialogOpen: true }), setLayoutDirection: (direction) => @@ -383,7 +412,18 @@ export const useRecipeStudioStore = create((set, get) => ({ } return buildAddedNodeState(state, "sampler", type, position, openDialog); }), - addSeedNode: (type, position, openDialog = true) => + addSeedNode: (type, position, openDialog = true) => { + const current = get(); + if (!current.executionLocked) { + // The reset below clears the block's upload uid and file list; queue + // its server-side directory for deletion after the next save. + const uid = seedUploadCleanupUid( + Object.values(current.configs).find((config) => config.kind === "seed"), + ); + if (uid) { + current.queueUploadCleanup(uid); + } + } set((state) => { if (state.executionLocked) { return state; @@ -413,6 +453,8 @@ export const useRecipeStudioStore = create((set, get) => ({ hf_token: "", hf_endpoint: "https://huggingface.co", local_file_name: "", + unstructured_upload_uid: + nextSourceType === "unstructured" ? makeUnstructuredUploadUid() : "", unstructured_file_ids: [], unstructured_file_names: [], unstructured_file_sizes: [], @@ -446,7 +488,8 @@ export const useRecipeStudioStore = create((set, get) => ({ activeConfigId: existing.id, dialogOpen: openDialog, }; - }), + }); + }, addLlmNode: (type, position, openDialog = true) => set((state) => { if (state.executionLocked) { @@ -699,6 +742,9 @@ export const useRecipeStudioStore = create((set, get) => ({ dialogOpen: false, sheetView: "root", fitViewTick: state.fitViewTick + 1, + // Queued cleanups belong to the previous recipe; draining them after + // a save of this one could delete files its saved payload still uses. + pendingUploadCleanups: [], })), setAuxNodePosition: (id, position) => set((state) => { @@ -786,6 +832,17 @@ export const useRecipeStudioStore = create((set, get) => ({ set(applyUpdate); }, onNodesChange: (changes) => { + const current = get(); + if (!current.executionLocked) { + for (const change of changes) { + if (change.type === "remove") { + const uid = seedUploadCleanupUid(current.configs[change.id]); + if (uid) { + current.queueUploadCleanup(uid); + } + } + } + } const applyNodesChange = (state: RecipeStudioState) => { if (state.executionLocked) { return state; diff --git a/studio/frontend/src/features/recipe-studio/types/index.ts b/studio/frontend/src/features/recipe-studio/types/index.ts index b8ed13f70b..9231c5f7a3 100644 --- a/studio/frontend/src/features/recipe-studio/types/index.ts +++ b/studio/frontend/src/features/recipe-studio/types/index.ts @@ -340,6 +340,8 @@ export type SeedConfig = { hf_token?: string; hf_endpoint?: string; local_file_name?: string; + // ui-only: stable per-block id for uploads, since node ids collide across imports + unstructured_upload_uid?: string; unstructured_file_ids?: string[]; unstructured_file_names?: string[]; unstructured_file_sizes?: number[]; diff --git a/studio/frontend/src/features/recipe-studio/utils/config-factories.ts b/studio/frontend/src/features/recipe-studio/utils/config-factories.ts index 76fc2c38ee..d47bac8858 100644 --- a/studio/frontend/src/features/recipe-studio/utils/config-factories.ts +++ b/studio/frontend/src/features/recipe-studio/utils/config-factories.ts @@ -20,6 +20,46 @@ import type { } from "../types"; import { nextName } from "./naming"; +export function makeUnstructuredUploadUid(): string { + if (typeof globalThis.crypto?.randomUUID === "function") { + return globalThis.crypto.randomUUID().replace(/-/g, "").toLowerCase(); + } + if (typeof globalThis.crypto?.getRandomValues === "function") { + const bytes = new Uint8Array(16); + globalThis.crypto.getRandomValues(bytes); + return Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join( + "", + ); + } + let uid = ""; + while (uid.length < 32) { + uid += Math.floor(Math.random() * 0x100000000) + .toString(16) + .padStart(8, "0"); + } + return uid.slice(0, 32); +} + +export function resolveUnstructuredUploadBlockId({ + configId, + uploadUid, + generatedUploadUid, + unstructuredFileCount, +}: { + configId: string; + uploadUid: string; + generatedUploadUid: string | null; + unstructuredFileCount: number; +}): string { + if (uploadUid) { + return uploadUid; + } + if (generatedUploadUid) { + return generatedUploadUid; + } + return unstructuredFileCount > 0 ? configId : ""; +} + export function makeSamplerConfig( id: string, samplerType: SamplerType, @@ -368,6 +408,9 @@ export function makeSeedConfig( hf_token: "", hf_endpoint: "https://huggingface.co", local_file_name: "", + ...(seedSourceType === "unstructured" + ? { unstructured_upload_uid: makeUnstructuredUploadUid() } + : {}), unstructured_file_ids: [], unstructured_file_names: [], unstructured_file_sizes: [], diff --git a/studio/frontend/src/features/recipe-studio/utils/import/importer.ts b/studio/frontend/src/features/recipe-studio/utils/import/importer.ts index ac881d0373..54df2ffd50 100644 --- a/studio/frontend/src/features/recipe-studio/utils/import/importer.ts +++ b/studio/frontend/src/features/recipe-studio/utils/import/importer.ts @@ -16,11 +16,7 @@ import type { } from "../../types"; import { buildEdges } from "./edges"; import { isRecord, parseJson, readString } from "./helpers"; -import { - parseColumn, - parseModelConfig, - parseModelProvider, -} from "./parsers"; +import { parseColumn, parseModelConfig, parseModelProvider } from "./parsers"; import { parseSeedConfig } from "./parsers/seed-config-parser"; import { buildNodes, parseUi } from "./ui"; import type { ImportResult } from "./types"; @@ -43,6 +39,7 @@ type UiInput = { seed_drop_columns?: unknown; seed_preview_rows?: unknown; local_file_name?: unknown; + unstructured_upload_uid?: unknown; unstructured_file_ids?: unknown; unstructured_file_names?: unknown; unstructured_file_sizes?: unknown; @@ -51,6 +48,10 @@ type UiInput = { advanced_open_by_node?: unknown; }; +type ImportRecipePayloadOptions = { + preserveUnstructuredUploads?: boolean; +}; + type UiMarkdownNoteNode = { name: string; markdown: string; @@ -90,7 +91,7 @@ function parseProcessors(input: unknown): RecipeProcessorConfig[] { ? templateRaw : isRecord(templateRaw) ? JSON.stringify(templateRaw, null, 2) - : "{\n \"text\": \"{{ column_name }}\"\n}"; + : '{\n "text": "{{ column_name }}"\n}'; processors.push({ id: `p${index + 1}`, // biome-ignore lint/style/useNamingConvention: api schema @@ -135,9 +136,7 @@ function parseSeedDropColumns(input: unknown): string[] { return Array.from(values); } -function parseMcpProviders( - input: unknown, -): Map { +function parseMcpProviders(input: unknown): Map { const providers = new Map(); if (!Array.isArray(input)) { return providers; @@ -156,13 +155,12 @@ function parseMcpProviders( const args = Array.isArray(item.args) ? item.args.map((value) => String(value)) : []; - const envPairs = - isRecord(item.env) - ? Object.entries(item.env).map(([key, value]) => ({ - key: String(key), - value: String(value), - })) - : []; + const envPairs = isRecord(item.env) + ? Object.entries(item.env).map(([key, value]) => ({ + key: String(key), + value: String(value), + })) + : []; providers.set(name, { id: `mcp-${index + 1}`, name, @@ -209,7 +207,8 @@ function parseToolConfigs(input: unknown): Map { allow_tools: allowTools, // biome-ignore lint/style/useNamingConvention: api schema max_tool_call_turns: - item.max_tool_call_turns === null || item.max_tool_call_turns === undefined + item.max_tool_call_turns === null || + item.max_tool_call_turns === undefined ? "5" : String(item.max_tool_call_turns), // biome-ignore lint/style/useNamingConvention: api schema @@ -257,7 +256,9 @@ function parseUiMarkdownNoteNodes(input: unknown): UiMarkdownNoteNode[] { return noteNodes; } -function parseUiToolProfileNodes(input: unknown): Map> { +function parseUiToolProfileNodes( + input: unknown, +): Map> { const toolProfiles = new Map>(); if (!Array.isArray(input)) { return toolProfiles; @@ -312,9 +313,15 @@ function parseAdvancedOpenByNode(input: unknown): Record { return out; } -type AdvancedOpenConfig = LlmConfig | SamplerConfig | SeedConfig | ValidatorConfig; +type AdvancedOpenConfig = + | LlmConfig + | SamplerConfig + | SeedConfig + | ValidatorConfig; -function isAdvancedOpenConfig(config: NodeConfig): config is AdvancedOpenConfig { +function isAdvancedOpenConfig( + config: NodeConfig, +): config is AdvancedOpenConfig { return ( config.kind === "llm" || config.kind === "sampler" || @@ -350,7 +357,8 @@ function buildToolProfileConfig( .map((providerName) => mcpProvidersByName.get(providerName)) .flatMap((provider) => (provider ? [cloneMcpProvider(provider)] : [])), // biome-ignore lint/style/useNamingConvention: ui schema - fetched_tools_by_provider: fetchedToolsByProfileName.get(canonical.tool_alias) ?? {}, + fetched_tools_by_provider: + fetchedToolsByProfileName.get(canonical.tool_alias) ?? {}, // biome-ignore lint/style/useNamingConvention: api schema allow_tools: [...(canonical.allow_tools ?? [])], // biome-ignore lint/style/useNamingConvention: api schema @@ -360,7 +368,10 @@ function buildToolProfileConfig( }; } -export function importRecipePayload(input: string): ImportResult { +export function importRecipePayload( + input: string, + options: ImportRecipePayloadOptions = {}, +): ImportResult { const parsed = parseJson(input); if (!parsed.data || !isRecord(parsed.data)) { return { @@ -369,9 +380,9 @@ export function importRecipePayload(input: string): ImportResult { }; } - const recipe = (isRecord(parsed.data.recipe) - ? parsed.data.recipe - : parsed.data) as RecipeInput; + const recipe = ( + isRecord(parsed.data.recipe) ? parsed.data.recipe : parsed.data + ) as RecipeInput; const ui = isRecord(parsed.data.ui) ? (parsed.data.ui as UiInput) : null; if (!Array.isArray(recipe.columns)) { @@ -410,21 +421,36 @@ export function importRecipePayload(input: string): ImportResult { .map((row) => ({ ...row })) : undefined; const uiLocalFileName = readString(ui?.local_file_name) ?? undefined; - // Preserve file IDs/names from saved recipes (cleared at share time by sanitizeSeedForShare) - const uiUnstructuredFileIds: string[] = Array.isArray(ui?.unstructured_file_ids) - ? (ui.unstructured_file_ids as string[]).filter((v): v is string => typeof v === "string") - : []; - const uiUnstructuredFileNames: string[] = Array.isArray(ui?.unstructured_file_names) - ? (ui.unstructured_file_names as string[]).filter((v): v is string => typeof v === "string") - : []; - const uiUnstructuredFileSizes: number[] = Array.isArray(ui?.unstructured_file_sizes) - ? (ui.unstructured_file_sizes as number[]).filter((v): v is number => typeof v === "number") - : []; + const preserveUnstructuredUploads = + options.preserveUnstructuredUploads === true; + const uiUnstructuredUploadUid = preserveUnstructuredUploads + ? (readString(ui?.unstructured_upload_uid) ?? undefined) + : undefined; + const uiUnstructuredFileIds: string[] = + preserveUnstructuredUploads && Array.isArray(ui?.unstructured_file_ids) + ? (ui.unstructured_file_ids as string[]).filter( + (v): v is string => typeof v === "string", + ) + : []; + const uiUnstructuredFileNames: string[] = + preserveUnstructuredUploads && Array.isArray(ui?.unstructured_file_names) + ? (ui.unstructured_file_names as string[]).filter( + (v): v is string => typeof v === "string", + ) + : []; + const uiUnstructuredFileSizes: number[] = + preserveUnstructuredUploads && Array.isArray(ui?.unstructured_file_sizes) + ? (ui.unstructured_file_sizes as number[]).filter( + (v): v is number => typeof v === "number", + ) + : []; const uiUnstructuredChunkSize = readStringNumber(ui?.unstructured_chunk_size); const uiUnstructuredChunkOverlap = readStringNumber( ui?.unstructured_chunk_overlap, ); - const uiAdvancedOpenByNode = parseAdvancedOpenByNode(ui?.advanced_open_by_node); + const uiAdvancedOpenByNode = parseAdvancedOpenByNode( + ui?.advanced_open_by_node, + ); const uiMarkdownNotes = parseUiMarkdownNoteNodes(ui?.nodes); const uiToolProfilesByName = parseUiToolProfileNodes(ui?.nodes); @@ -459,11 +485,13 @@ export function importRecipePayload(input: string): ImportResult { : payloadSeedDropColumns, seed_preview_rows: uiSeedPreviewRows, local_file_name: uiLocalFileName, + unstructuredUploadUid: uiUnstructuredUploadUid, unstructuredFileIds: uiUnstructuredFileIds, unstructuredFileNames: uiUnstructuredFileNames, unstructuredFileSizes: uiUnstructuredFileSizes, unstructured_chunk_size: uiUnstructuredChunkSize, unstructured_chunk_overlap: uiUnstructuredChunkOverlap, + preserveUnstructuredUploads, }); if (seedConfig) { applyAdvancedOpen(seedConfig, uiAdvancedOpenByNode); @@ -567,12 +595,7 @@ export function importRecipePayload(input: string): ImportResult { const { layouts, auxNodes, edges: uiEdges, layoutDirection } = parseUi(ui); const resolvedLayoutDirection = layoutDirection ?? "LR"; const nodes = buildNodes(configs, layouts); - const edges = buildEdges( - configs, - nameToId, - uiEdges, - resolvedLayoutDirection, - ); + const edges = buildEdges(configs, nameToId, uiEdges, resolvedLayoutDirection); const auxNodePositions = Object.fromEntries( auxNodes.flatMap((item) => { const llmId = nameToId.get(item.llm); @@ -583,10 +606,7 @@ export function importRecipePayload(input: string): ImportResult { }), ); - const maxY = nodes.reduce( - (acc, node) => Math.max(acc, node.position.y), - 0, - ); + const maxY = nodes.reduce((acc, node) => Math.max(acc, node.position.y), 0); return { errors: [], diff --git a/studio/frontend/src/features/recipe-studio/utils/import/parsers/seed-config-parser.ts b/studio/frontend/src/features/recipe-studio/utils/import/parsers/seed-config-parser.ts index 21eadb3195..939205fe6d 100644 --- a/studio/frontend/src/features/recipe-studio/utils/import/parsers/seed-config-parser.ts +++ b/studio/frontend/src/features/recipe-studio/utils/import/parsers/seed-config-parser.ts @@ -197,17 +197,26 @@ export function parseSeedConfig( seed_drop_columns?: string[]; seed_preview_rows?: Record[]; local_file_name?: string; + unstructuredUploadUid?: string; unstructuredFileIds?: string[]; unstructuredFileNames?: string[]; unstructuredFileSizes?: number[]; unstructured_chunk_size?: string; unstructured_chunk_overlap?: string; + preserveUnstructuredUploads?: boolean; }, ): SeedConfig | null { if (!seedConfigRaw) { return null; } - const parsed = parseSeedSettings(seedConfigRaw); + const parsed = { ...parseSeedSettings(seedConfigRaw) }; + if ( + parsed.seed_source_type === "unstructured" && + options?.preserveUnstructuredUploads !== true + ) { + parsed.hf_path = ""; + parsed.resolved_paths = []; + } let sourceType: SeedSourceType = "hf"; if (parsed.seed_source_type === "hf") { sourceType = "hf"; @@ -230,6 +239,9 @@ export function parseSeedConfig( ...(options?.local_file_name !== undefined ? { local_file_name: options.local_file_name } : {}), + ...(options?.unstructuredUploadUid + ? { unstructured_upload_uid: options.unstructuredUploadUid } + : {}), ...(options?.unstructuredFileIds !== undefined ? { unstructured_file_ids: options.unstructuredFileIds } : {}), diff --git a/studio/frontend/src/features/recipe-studio/utils/payload/build-payload.ts b/studio/frontend/src/features/recipe-studio/utils/payload/build-payload.ts index 34c3c34274..9b2ad5b2b5 100644 --- a/studio/frontend/src/features/recipe-studio/utils/payload/build-payload.ts +++ b/studio/frontend/src/features/recipe-studio/utils/payload/build-payload.ts @@ -440,6 +440,9 @@ export function buildRecipePayload( unstructured_file_names: firstSeed.unstructured_file_names, unstructured_file_sizes: firstSeed.unstructured_file_sizes, }), + ...(firstSeed?.unstructured_upload_uid?.trim() && { + unstructured_upload_uid: firstSeed.unstructured_upload_uid, + }), ...(firstSeed && firstSeed.unstructured_chunk_size !== undefined && { unstructured_chunk_size: firstSeed.unstructured_chunk_size, diff --git a/studio/frontend/src/features/recipe-studio/utils/payload/types.ts b/studio/frontend/src/features/recipe-studio/utils/payload/types.ts index 902ea796b4..763e68c1fb 100644 --- a/studio/frontend/src/features/recipe-studio/utils/payload/types.ts +++ b/studio/frontend/src/features/recipe-studio/utils/payload/types.ts @@ -71,6 +71,8 @@ export type RecipePayload = { seed_preview_rows?: Record[]; local_file_name?: string; // biome-ignore lint/style/useNamingConvention: api schema + unstructured_upload_uid?: string; + // biome-ignore lint/style/useNamingConvention: api schema unstructured_file_ids?: string[]; // biome-ignore lint/style/useNamingConvention: api schema unstructured_file_names?: string[]; From 1b825213ea2ffe4774404f27795ad287634f6681 Mon Sep 17 00:00:00 2001 From: Michael Han <107991372+shimmyshimmer@users.noreply.github.com> Date: Thu, 9 Jul 2026 00:16:05 -0700 Subject: [PATCH 009/372] Stabilize floating monitor drag (#6984) * Stabilize floating monitor drag * Restore floating monitor exit animation * Harden Windows Studio smoke checks * Keep API menu badge removed * Apply no-build-tools env overrides in-script The runner does not apply step-level env keys containing parentheses, so ProgramFiles(x86) kept its real value and Find-VsBuildTools still detected VS through vswhere. Set the overrides inside each pwsh step instead; child processes inherit them. The resolver step moves to pwsh because bash cannot export a variable named ProgramFiles(x86). * Reset chat UI session without a second browser context macOS runs Chromium with --single-process, where closing the last context tears down the whole browser, so the shutdown re-login died with TargetClosedError on new_page. Clear cookies and swap pages inside the same context instead, opening the replacement page before closing the old one. * Keep the no-build-tools Path filtered across session refreshes install.ps1's Refresh-SessionPath and setup.ps1's Refresh-Environment rebuild the session Path from the Machine and User registry scopes, so the process-level filter could be undone mid-install and re-expose CMake. Filter those scopes in the Prepare step with normalized dir matching and restore them in cleanup. * Drop stale localStorage auth tokens before re-login Auth tokens live in localStorage, not cookies, and the login guest guard redirects on their mere presence. Remove them during the session reset so the /login navigation is deterministic instead of relying on the tolerated redirect bounce. --- .../studio-windows-inference-smoke.yml | 161 +++++++++---- .../frontend/src/components/app-sidebar.tsx | 17 +- .../src/components/floating-monitor.tsx | 223 ++++++++++-------- studio/frontend/src/features/chat/index.ts | 1 + .../frontend/src/features/settings/index.ts | 1 + studio/frontend/src/i18n/locales/en.ts | 1 - studio/frontend/src/i18n/locales/ja.ts | 1 - studio/frontend/src/i18n/locales/pt-br.ts | 1 - studio/frontend/src/i18n/locales/zh-CN.ts | 1 - tests/studio/playwright_chat_ui.py | 36 ++- 10 files changed, 277 insertions(+), 166 deletions(-) diff --git a/.github/workflows/studio-windows-inference-smoke.yml b/.github/workflows/studio-windows-inference-smoke.yml index 0bc216d65a..dbb0f9ea6f 100644 --- a/.github/workflows/studio-windows-inference-smoke.yml +++ b/.github/workflows/studio-windows-inference-smoke.yml @@ -1334,42 +1334,75 @@ jobs: try { Add-MpPreference -ExclusionPath $p -ErrorAction Stop } catch { } } - - name: Hide Visual Studio + CMake (simulate a host with no build tools) + - name: Prepare no-build-tools simulation shell: pwsh run: | $ErrorActionPreference = 'Stop' - # A Program Files dir can hold a transient handle (Defender / MSBuild node) - # so Rename-Item intermittently fails with "Access is denied"; retry to ride it out. - function Rename-WithRetry($Path, $NewName) { - for ($i = 1; $i -le 6; $i++) { - try { Rename-Item -LiteralPath $Path -NewName $NewName -ErrorAction Stop; return } - catch { if ($i -eq 6) { throw }; Start-Sleep -Seconds 3 } + $root = Join-Path $env:GITHUB_WORKSPACE 'no-build-tools' + $pf = Join-Path $root 'ProgramFiles' + $pfx86 = Join-Path $root 'ProgramFilesx86' + New-Item -ItemType Directory -Force -Path $pf, $pfx86 | Out-Null + + $blocked = [System.Collections.Generic.HashSet[string]]::new([StringComparer]::OrdinalIgnoreCase) + foreach ($tool in @('cmake', 'cl.exe')) { + foreach ($cmd in (Get-Command $tool -All -ErrorAction SilentlyContinue)) { + if ($cmd.Source) { + $dir = Split-Path -Parent $cmd.Source + if ($dir) { + [void] $blocked.Add( + [Environment]::ExpandEnvironmentVariables($dir).Trim().Trim('"').TrimEnd('\')) + } + } } } - # Rename the Visual Studio install roots (incl. the Installer that holds - # vswhere.exe) so Find-VsBuildTools' vswhere + filesystem scan both miss. - foreach ($d in @("$env:ProgramFiles\Microsoft Visual Studio", "${env:ProgramFiles(x86)}\Microsoft Visual Studio")) { - if (Test-Path -LiteralPath $d) { - Rename-WithRetry $d ((Split-Path $d -Leaf) + '.vsoff') - Write-Host "Hid VS: $d" - } + # Normalized comparison so registry spellings (trailing slash, + # unexpanded %VAR%) still match. + function Test-Blocked([string]$p) { + $n = [Environment]::ExpandEnvironmentVariables($p).Trim().Trim('"').TrimEnd('\') + return $blocked.Contains($n) } - # Surgically rename each cmake executable on PATH (not its parent dir -- - # cmake can share a dir with other shims) so Get-Command cmake fails. - $hidden = @() - foreach ($c in (Get-Command cmake -All -ErrorAction SilentlyContinue)) { - if ($c.Source -and (Test-Path -LiteralPath $c.Source)) { - Rename-WithRetry $c.Source ((Split-Path $c.Source -Leaf) + '.off') - $hidden += $c.Source - Write-Host "Hid cmake: $($c.Source)" - } + + $pathParts = $env:Path -split [IO.Path]::PathSeparator | + Where-Object { $_ -and -not (Test-Blocked $_) } + $noBuildToolsPath = $pathParts -join [IO.Path]::PathSeparator + + # install.ps1's Refresh-SessionPath and setup.ps1's Refresh-Environment + # rebuild the session Path from these scopes mid-install, so filter + # them too. Originals are saved for the cleanup step. + foreach ($scope in @('Machine', 'User')) { + $orig = [Environment]::GetEnvironmentVariable('Path', $scope) + if (-not $orig) { continue } + Set-Content -LiteralPath (Join-Path $root "orig-path-$scope.txt") -Value $orig -NoNewline + $kept = ($orig -split ';' | Where-Object { $_ -and -not (Test-Blocked $_) }) -join ';' + [Environment]::SetEnvironmentVariable('Path', $kept, $scope) + Write-Host "Filtered $scope Path scope." + } + + "NO_BUILD_TOOLS_PROGRAMFILES=$pf" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 + "NO_BUILD_TOOLS_PROGRAMFILES_X86=$pfx86" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 + "NO_BUILD_TOOLS_PATH<&1 | Tee-Object -FilePath logs/install.log @@ -1480,19 +1517,19 @@ jobs: [ -n "$CONTENT" ] && [ "$CONTENT" != "null" ] || { echo "::error::empty completion"; exit 1; } echo "Inference OK without Visual Studio: $CONTENT" - - name: Restore Visual Studio + CMake + - name: Clean no-build-tools simulation if: always() shell: pwsh run: | - foreach ($d in @("$env:ProgramFiles\Microsoft Visual Studio", "${env:ProgramFiles(x86)}\Microsoft Visual Studio")) { - $off = "$d.vsoff" - if (Test-Path -LiteralPath $off) { Rename-Item -LiteralPath $off -NewName (Split-Path $d -Leaf); Write-Host "Restored $d" } - } - if ($env:HIDDEN_CMAKE) { - foreach ($src in ($env:HIDDEN_CMAKE -split '\|')) { - if ($src -and (Test-Path -LiteralPath "$src.off")) { Rename-Item -LiteralPath "$src.off" -NewName (Split-Path $src -Leaf) } + $root = Join-Path $env:GITHUB_WORKSPACE 'no-build-tools' + foreach ($scope in @('Machine', 'User')) { + $saved = Join-Path $root "orig-path-$scope.txt" + if (Test-Path -LiteralPath $saved) { + [Environment]::SetEnvironmentVariable('Path', (Get-Content -LiteralPath $saved -Raw), $scope) + Write-Host "Restored $scope Path scope." } } + Remove-Item -LiteralPath $root -Recurse -Force -ErrorAction SilentlyContinue - name: Stop Studio if: always() @@ -1540,21 +1577,34 @@ jobs: with: python-version: '3.12' - - name: Hide Visual Studio + - name: Prepare no-build-tools simulation shell: pwsh run: | $ErrorActionPreference = 'Stop' - # Retry the rename: a Program Files dir can hold a transient handle that - # makes Rename-Item intermittently fail with "Access is denied". - function Rename-WithRetry($Path, $NewName) { - for ($i = 1; $i -le 6; $i++) { - try { Rename-Item -LiteralPath $Path -NewName $NewName -ErrorAction Stop; return } - catch { if ($i -eq 6) { throw }; Start-Sleep -Seconds 3 } + $root = Join-Path $env:GITHUB_WORKSPACE 'no-build-tools' + $pf = Join-Path $root 'ProgramFiles' + $pfx86 = Join-Path $root 'ProgramFilesx86' + New-Item -ItemType Directory -Force -Path $pf, $pfx86 | Out-Null + + $blocked = [System.Collections.Generic.HashSet[string]]::new([StringComparer]::OrdinalIgnoreCase) + foreach ($tool in @('cmake', 'cl.exe')) { + foreach ($cmd in (Get-Command $tool -All -ErrorAction SilentlyContinue)) { + if ($cmd.Source) { + $dir = Split-Path -Parent $cmd.Source + if ($dir) { [void] $blocked.Add($dir) } + } } } - foreach ($d in @("$env:ProgramFiles\Microsoft Visual Studio", "${env:ProgramFiles(x86)}\Microsoft Visual Studio")) { - if (Test-Path -LiteralPath $d) { Rename-WithRetry $d ((Split-Path $d -Leaf) + '.vsoff'); Write-Host "Hid VS: $d" } - } + + $pathParts = $env:Path -split [IO.Path]::PathSeparator | + Where-Object { $_ -and -not $blocked.Contains($_) } + $noBuildToolsPath = $pathParts -join [IO.Path]::PathSeparator + + "NO_BUILD_TOOLS_PROGRAMFILES=$pf" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 + "NO_BUILD_TOOLS_PROGRAMFILES_X86=$pfx86" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 + "NO_BUILD_TOOLS_PATH< /tmp/resolve.json || { - echo "::error::resolver exited non-zero"; cat /tmp/resolve.json || true; exit 1; } - cat /tmp/resolve.json - echo "Prebuilt resolver ran with no Visual Studio present." + if ($LASTEXITCODE -ne 0) { Write-Host "::error::pip install huggingface_hub failed"; exit 1 } + python studio/install_llama_prebuilt.py --resolve-prebuilt latest --output-format json > resolve.json + if ($LASTEXITCODE -ne 0) { + Write-Host "::error::resolver exited non-zero" + if (Test-Path resolve.json) { Get-Content resolve.json } + exit 1 + } + Get-Content resolve.json + Write-Host "Prebuilt resolver ran with no Visual Studio present." - - name: Restore Visual Studio + - name: Clean no-build-tools simulation if: always() shell: pwsh run: | - foreach ($d in @("$env:ProgramFiles\Microsoft Visual Studio", "${env:ProgramFiles(x86)}\Microsoft Visual Studio")) { - $off = "$d.vsoff" - if (Test-Path -LiteralPath $off) { Rename-Item -LiteralPath $off -NewName (Split-Path $d -Leaf); Write-Host "Restored $d" } - } + Remove-Item -LiteralPath (Join-Path $env:GITHUB_WORKSPACE 'no-build-tools') -Recurse -Force -ErrorAction SilentlyContinue # ── folded from studio-setup-ps1-vs2026.yml: setup.ps1 unit tests + real-VS detection + vcredist ── pester: diff --git a/studio/frontend/src/components/app-sidebar.tsx b/studio/frontend/src/components/app-sidebar.tsx index 823e420869..1a74b38524 100644 --- a/studio/frontend/src/components/app-sidebar.tsx +++ b/studio/frontend/src/components/app-sidebar.tsx @@ -81,7 +81,6 @@ import { TestTube01Icon, ZapIcon, } from "@hugeicons/core-free-icons"; -import { listStoredChatThreads } from "@/features/chat/utils/chat-history-storage"; import { Tooltip, TooltipContent, @@ -97,6 +96,7 @@ import { createChatProject, deleteChatProject, deleteChatItem, + listStoredChatThreads, moveChatItemToProject, renameChatItem, renameChatProject, @@ -582,7 +582,14 @@ export function AppSidebar() { useEffect(() => { if (!pendingRename) return; const match = allChatItems.find((i) => i.id === pendingRename.id); - if (match && match.title === pendingRename.title) setPendingRename(null); + if (!match || match.title !== pendingRename.title) return; + queueMicrotask(() => { + setPendingRename((current) => + current?.id === pendingRename.id && current.title === pendingRename.title + ? null + : current, + ); + }); }, [allChatItems, pendingRename]); const [creatingProject, setCreatingProject] = useState(false); const [projectNameDraft, setProjectNameDraft] = useState(""); @@ -680,12 +687,6 @@ export function AppSidebar() { useState(null); const [deleteProjectFiles, setDeleteProjectFiles] = useState(false); - useEffect(() => { - if (confirmingDelete?.kind !== "project") { - setDeleteProjectFiles(false); - } - }, [confirmingDelete]); - async function commitDelete() { const target = confirmingDelete; if (!target) return; diff --git a/studio/frontend/src/components/floating-monitor.tsx b/studio/frontend/src/components/floating-monitor.tsx index bce4bf2831..0a51875de9 100644 --- a/studio/frontend/src/components/floating-monitor.tsx +++ b/studio/frontend/src/components/floating-monitor.tsx @@ -3,27 +3,35 @@ import { Button } from "@/components/ui/button"; import { Progress } from "@/components/ui/progress"; -import { useMonitorOverlayStore } from "@/features/settings/stores/monitor-overlay-store"; +import { useMonitorOverlayStore } from "@/features/settings"; import { useSystemInfo } from "@/hooks/use-system"; import { useT } from "@/i18n"; import { cn } from "@/lib/utils"; import { CpuIcon, GripVerticalIcon, XIcon } from "lucide-react"; -import { motion } from "motion/react"; -import { useRef } from "react"; +import { AnimatePresence, motion, useDragControls } from "motion/react"; +import { type PointerEvent, useMemo, useState } from "react"; function clampPercent(value: number): number { return Math.max(0, Math.min(100, value)); } function usageIndicatorClass(percent: number): string { - if (percent >= 90) return "bg-destructive"; - if (percent >= 70) return "bg-amber-500"; + if (percent >= 90) { + return "bg-destructive"; + } + if (percent >= 70) { + return "bg-amber-500"; + } return "bg-primary"; } function usageTextClass(percent: number): string { - if (percent >= 90) return "text-destructive"; - if (percent >= 70) return "text-amber-600 dark:text-amber-400"; + if (percent >= 90) { + return "text-destructive"; + } + if (percent >= 70) { + return "text-amber-600 dark:text-amber-400"; + } return "text-primary"; } @@ -39,9 +47,18 @@ export function FloatingMonitor() { const { isOpen, setIsOpen } = useMonitorOverlayStore(); const systemInfo = useSystemInfo({ enabled: isOpen, pollMs: 5000 }); - const constraintsRef = useRef(null); + const [constraintsElement, setConstraintsElement] = + useState(null); + const constraintsRef = useMemo( + () => ({ current: constraintsElement }), + [constraintsElement], + ); + const dragControls = useDragControls(); - if (!isOpen) return null; + function startDrag(event: PointerEvent) { + event.preventDefault(); + dragControls.start(event); + } const ramTotal = systemInfo.memory?.total_gb ?? 0; const ramAvailable = systemInfo.memory?.available_gb ?? 0; @@ -64,99 +81,109 @@ export function FloatingMonitor() { const hasGpu = (systemInfo.gpu?.available ?? false) && devices.length > 0; return ( -
- -
-
- - - {t("settings.resources.liveMonitor.title")} - -
-
-
- -
- - -
-
- - + {isOpen && ( +
-
-
- {t("settings.resources.liveMonitor.ram")} - - {Math.round(ramPercent)}% - -
-
- {formatGiB(ramUsed)} / {formatGiB(ramTotal)} -
- -
- - {hasGpu && ( -
-
- - {t("settings.resources.liveMonitor.vram")}{" "} - {devices.length > 1 - ? `(${devices.length} GPUs)` - : `(${devices[0].name ?? "GPU"})`} + +
+
+ + + {t("settings.resources.liveMonitor.title")} - +
+
- {Math.round(vramPercent)}% - + +
+ +
-
- {formatGiB(vramUsed)} / {formatGiB(vramTotal)} -
-
- )} - - -
+ + +
+
+ {t("settings.resources.liveMonitor.ram")} + + {Math.round(ramPercent)}% + +
+
+ {formatGiB(ramUsed)} / {formatGiB(ramTotal)} +
+ +
+ + {hasGpu && ( +
+
+ + {t("settings.resources.liveMonitor.vram")}{" "} + {devices.length > 1 + ? `(${devices.length} GPUs)` + : `(${devices[0].name ?? "GPU"})`} + + + {Math.round(vramPercent)}% + +
+
+ {formatGiB(vramUsed)} / {formatGiB(vramTotal)} +
+ +
+ )} +
+
+
+ )} + ); } diff --git a/studio/frontend/src/features/chat/index.ts b/studio/frontend/src/features/chat/index.ts index 7cd9611c71..d070ed15de 100644 --- a/studio/frontend/src/features/chat/index.ts +++ b/studio/frontend/src/features/chat/index.ts @@ -36,6 +36,7 @@ export { ChatSearchDialog } from "./components/chat-search-dialog"; export { setTrainingCompareHandoff } from "./lib/training-compare-handoff"; export type { ProjectRecord } from "./types"; export { clearAllChats, countAllChats } from "./utils/clear-all-chats"; +export { listStoredChatThreads } from "./utils/chat-history-storage"; export { ArtifactCard } from "./artifacts/artifact-card"; export { useChatArtifactsStore, diff --git a/studio/frontend/src/features/settings/index.ts b/studio/frontend/src/features/settings/index.ts index 364ca1611f..3fefd8c63a 100644 --- a/studio/frontend/src/features/settings/index.ts +++ b/studio/frontend/src/features/settings/index.ts @@ -7,6 +7,7 @@ export { savePersonalization, } from "./api/personalization"; export { setTheme, useTheme } from "./stores/theme-store"; +export { useMonitorOverlayStore } from "./stores/monitor-overlay-store"; export type { Personalization, PersonalizationAppearance, diff --git a/studio/frontend/src/i18n/locales/en.ts b/studio/frontend/src/i18n/locales/en.ts index a9b5d839b3..e0cb8030ae 100644 --- a/studio/frontend/src/i18n/locales/en.ts +++ b/studio/frontend/src/i18n/locales/en.ts @@ -409,7 +409,6 @@ export const en = { description: "Access Unsloth via the OpenAI-compatible API.", readDocs: "Read the API docs", noAccess: "No API access yet.", - newBadge: "New", accessTokens: "Access tokens", loadError: "Couldn't load API access.", createError: "Couldn't create access token.", diff --git a/studio/frontend/src/i18n/locales/ja.ts b/studio/frontend/src/i18n/locales/ja.ts index f2020f3cfb..08b8d4f4d3 100644 --- a/studio/frontend/src/i18n/locales/ja.ts +++ b/studio/frontend/src/i18n/locales/ja.ts @@ -296,7 +296,6 @@ export const ja = { description: "OpenAI互換 API を介して Unsloth にアクセスします。", readDocs: "API ドキュメントを読む", noAccess: "まだ API アクセス権がありません。", - newBadge: "新規", accessTokens: "アクセストークン", loadError: "API アクセス権を読み込めませんでした。", createError: "アクセストークンを作成できませんでした。", diff --git a/studio/frontend/src/i18n/locales/pt-br.ts b/studio/frontend/src/i18n/locales/pt-br.ts index c261e4ed0c..494a98ec32 100644 --- a/studio/frontend/src/i18n/locales/pt-br.ts +++ b/studio/frontend/src/i18n/locales/pt-br.ts @@ -363,7 +363,6 @@ export const ptBR = { "Acesse o Unsloth por meio da API compatível com OpenAI.", readDocs: "Leia a documentação da API", noAccess: "Nenhum acesso à API ainda.", - newBadge: "Novo", accessTokens: "Tokens de acesso", loadError: "Não foi possível carregar o acesso à API.", createError: "Não foi possível criar o token de acesso.", diff --git a/studio/frontend/src/i18n/locales/zh-CN.ts b/studio/frontend/src/i18n/locales/zh-CN.ts index f6dc265fc7..dda8e017ab 100644 --- a/studio/frontend/src/i18n/locales/zh-CN.ts +++ b/studio/frontend/src/i18n/locales/zh-CN.ts @@ -267,7 +267,6 @@ export const zhCN = { description: "通过兼容 OpenAI 的 API 以编程方式访问 Unsloth。", readDocs: "阅读 API 文档", noAccess: "还没有 API 访问权限。", - newBadge: "新", accessTokens: "访问 token", loadError: "无法加载 API 访问权限。", createError: "无法创建访问 token。", diff --git a/tests/studio/playwright_chat_ui.py b/tests/studio/playwright_chat_ui.py index 71297f9043..0698d8e0d2 100644 --- a/tests/studio/playwright_chat_ui.py +++ b/tests/studio/playwright_chat_ui.py @@ -1264,11 +1264,37 @@ with sync_playwright() as p: # placeholder, and /api/health goes unreachable shortly after. # ───────────────────────────────────────────────────── step("Shutdown via account menu") - # Re-login with NEW2 for a valid /api/shutdown token (CLI rotation - # invalidated the old one). The stale token can make the SPA auth guard - # abort this goto with ERR_ABORTED, or redirect to the same /login URL - # ("interrupted by another navigation"); resolve on domcontentloaded and - # tolerate either -- the pw-field wait below confirms we are on /login. + # Start fresh after the CLI rotation invalidates this browser session. + # Stay in the SAME context: macOS Chromium runs --single-process, where + # closing the last context kills the browser and a second context cannot + # be created. Open the new page before closing the old one; the context + # init script covers the new page. + try: + ctx.clear_cookies() + except Exception as exc: + info(f"WARN clearing stale session cookies failed: {exc!r}") + # Auth tokens live in localStorage, and /login's guest guard redirects on + # their mere presence, so drop them before navigating. + try: + page.evaluate( + "['unsloth_auth_token', 'unsloth_auth_refresh_token']" + ".forEach((key) => localStorage.removeItem(key))" + ) + except Exception as exc: + info(f"WARN clearing stale auth tokens failed: {exc!r}") + _fresh_page = ctx.new_page() + _fresh_page.set_default_timeout(60_000) + _fresh_page.on("pageerror", lambda e: page_errors.append(str(e))) + _fresh_page.on("console", _on_console) + try: + page.close() + except Exception: + pass + page = _fresh_page + + # Re-login with NEW2 for a valid /api/shutdown token. Route changes can + # still abort or interrupt this navigation, so the field wait below is the + # final confirmation that we reached /login. _tolerated_nav = ("ERR_ABORTED", "interrupted by another navigation") try: page.goto(f"{BASE}/login", wait_until = "domcontentloaded", timeout = 60_000) From 8205d4c0819088a3c864fdd505ce1c1e6d72852d Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 9 Jul 2026 01:46:14 -0700 Subject: [PATCH 010/372] Retry the Studio UI shutdown re-login on transient goto timeout (#7027) * Retry the Studio UI shutdown re-login on transient goto timeout The Chat UI Playwright smoke intermittently failed at the pre-shutdown re-login: page.goto('/login') can hit a 60s TimeoutError on a slow runner even while the server is healthy, and the surrounding except only tolerated ERR_ABORTED / interrupted-navigation, so a plain timeout hard-failed the job. Wrap the re-login goto/wait/fill/submit in the same 3-attempt retry the change-password step already uses (recover_or_replace_page between tries, per-attempt fail screenshots, wait_for_health pre-gate). The composer wait stays outside the loop so a retry never re-navigates after login has set tokens (which would redirect to /chat via the guest guard); it remains the authoritative confirmation, so a genuinely broken login still fails. * Catch transient login-request failures and preserve error listeners on recovery Wait on the /api/auth/login POST inside the retry (via click_and_wait_for_response) so a transient 4xx/5xx is retried in-loop instead of surfacing only at the out-of-loop composer wait, matching the change-password step. When recover_or_replace_page swaps in a fresh page, re-attach the pageerror/console listeners so error tracking survives the replacement. --- tests/studio/playwright_chat_ui.py | 96 ++++++++++++++++++++++++++---- 1 file changed, 86 insertions(+), 10 deletions(-) diff --git a/tests/studio/playwright_chat_ui.py b/tests/studio/playwright_chat_ui.py index 0698d8e0d2..6a88b98c19 100644 --- a/tests/studio/playwright_chat_ui.py +++ b/tests/studio/playwright_chat_ui.py @@ -1296,16 +1296,92 @@ with sync_playwright() as p: # still abort or interrupt this navigation, so the field wait below is the # final confirmation that we reached /login. _tolerated_nav = ("ERR_ABORTED", "interrupted by another navigation") - try: - page.goto(f"{BASE}/login", wait_until = "domcontentloaded", timeout = 60_000) - except Exception as exc: - if not any(t in str(exc) for t in _tolerated_nav): - raise - info(f"goto /login interrupted ({exc!r}); password-field wait will confirm /login") - pw_field = page.locator("#password") - pw_field.wait_for(state = "visible", timeout = 60_000) - pw_field.fill(NEW2) - page.locator('button[type="submit"]').click() + # A slow CI runner can make this re-login navigation time out even with the + # server healthy, so retry the whole goto/wait/fill/submit sequence (mirrors + # the change-password retry above). wait_for_health is a diagnostic pre-gate. + wait_for_health(BASE, timeout = 30.0, info = info) + relogin_err: Exception | None = None + for _relogin_attempt in range(3): + try: + try: + page.goto(f"{BASE}/login", wait_until = "domcontentloaded", timeout = 60_000) + except Exception as exc: + if not any(t in str(exc) for t in _tolerated_nav): + raise + info(f"goto /login interrupted ({exc!r}); password-field wait will confirm /login") + pw_field = page.locator("#password") + pw_field.wait_for(state = "visible", timeout = 60_000) + pw_field.fill(NEW2) + # Wait on the login POST so a transient 4xx/5xx is caught and retried + # here, not swallowed until the out-of-loop composer wait. + status, _ = click_and_wait_for_response( + page, + url_substr = "/api/auth/login", + method = "POST", + do_click = lambda: page.locator('button[type="submit"]').click(), + timeout_ms = 30_000, + info = lambda m: print(f"[ui] {m}", flush = True), + ) + if status is not None and status >= 400: + raise AssertionError( + f"login POST returned {status}; see console_errors={console_errors[:1]!r}" + ) + relogin_err = None + break + except Exception as e: + relogin_err = e + try: + cur_url = page.url + except Exception: + cur_url = "" + print( + f"[ui] re-login attempt {_relogin_attempt + 1} failed: " + f"{type(e).__name__}: {str(e)[:200]}; page.url={cur_url}; " + f"page_errors={len(page_errors)} console_errors={len(console_errors)}", + flush = True, + ) + if console_errors: + print( + f"[ui] first console.error: {console_errors[0][:200]!r}", + flush = True, + ) + if page_errors: + print(f"[ui] first pageerror: {page_errors[0][:200]!r}", flush = True) + try: + shoot(f"18-relogin-attempt-{_relogin_attempt + 1}-fail") + except Exception: + pass + if _relogin_attempt < 2: + # ERR_NO_BUFFER_SPACE needs the OS to recover socket + # buffers; back off 5s then 15s before retrying. + if "ERR_NO_BUFFER_SPACE" in str(e): + backoff_s = 5 if _relogin_attempt == 0 else 15 + print( + f"[ui] ENOBUFS detected; sleeping {backoff_s}s " + f"before retry to let OS recover socket buffers...", + flush = True, + ) + time.sleep(backoff_s) + # Replace the page if it died; otherwise next iteration's + # page.goto() handles the reload. + old_page = page + page = recover_or_replace_page( + page, + ctx, + default_timeout_ms = 60_000, + info = lambda m: print(f"[ui] recovery: {m}", flush = True), + ) + # A freshly created replacement page loses the pageerror/console + # listeners; re-attach so error tracking survives recovery. + if page is not old_page: + page.on("pageerror", lambda e: page_errors.append(str(e))) + page.on("console", _on_console) + if relogin_err is not None: + raise relogin_err + # Composer mount confirms the rotated session is authenticated. Kept OUTSIDE the + # retry: the loop breaks right after submit, so we never re-goto /login once login + # has set tokens -- that would hit the guest guard, redirect to /chat, and make a + # merely-slow composer look like a broken login. composer = page.locator('textarea[aria-label="Message input"]') composer.wait_for(state = "visible", timeout = 60_000) shoot("18-relogin-with-NEW2") From 5e43c623b98affc23efbf9dbe71061de7c1706a2 Mon Sep 17 00:00:00 2001 From: Etherl <61019402+Etherll@users.noreply.github.com> Date: Thu, 9 Jul 2026 11:46:22 +0300 Subject: [PATCH 011/372] Fix FastSentenceTransformer Qwen embedding preprocessing (#6939) * Fix FastSentenceTransformer Qwen embedding preprocessing * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Document Transformer.load embedding modality fix for #6881 * Harden #6881 fix and add forwards/backwards-compatible regression tests * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fall back to Transformer constructor on legacy sentence-transformers without Hub-capable load * Mirror legacy sentence-transformers fallback in embedding-parity tripwire test * Tighten #6881 comments and docstrings * Skip embedding-parity test on CPU-only runners since FastSentenceTransformer requires CUDA * Honor the transformer module's saved subfolder when loading modules.json records a path for the Transformer module (root for decoder embedders like Qwen3-Embedding, 0_Transformer for the classic layout). Pooling/Normalize already load from their saved path; thread the same path into Transformer.load as subfolder so config and tokenizer resolve like stock ST. stays a no-op, so single-module models are unchanged. * Make embedding-parity test bf16-aware fp16 overflows to NaN on bf16-native embedders such as EmbeddingGemma (Gemma3), producing a false parity failure. Prefer bf16 when the GPU supports it so the tripwire can guard the full documented embedding matrix (Qwen3-Embedding, EmbeddingGemma, BGE-M3, all-MiniLM, GTE-ModernBERT), not just fp16-safe models. --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: danielhanchen --- ...t_sentence_transformer_embedding_parity.py | 122 ++++++++++++++++++ ...st_sentence_transformers_pinned_symbols.py | 38 ++++++ unsloth/models/sentence_transformer.py | 65 +++++++++- 3 files changed, 222 insertions(+), 3 deletions(-) create mode 100644 tests/python/test_fast_sentence_transformer_embedding_parity.py diff --git a/tests/python/test_fast_sentence_transformer_embedding_parity.py b/tests/python/test_fast_sentence_transformer_embedding_parity.py new file mode 100644 index 0000000000..252d4486a5 --- /dev/null +++ b/tests/python/test_fast_sentence_transformer_embedding_parity.py @@ -0,0 +1,122 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. +"""Regression guard for issue #6881: FastSentenceTransformer must preprocess text +like a stock SentenceTransformer for decoder embedding models. ST 5.x infers a +"message" modality for chat-template models (e.g. Qwen/Qwen3-Embedding), so building +via `Transformer(model_name, ...)` chat-wraps inputs and degrades embeddings; +`_create_transformer_module` uses `Transformer.load(...)` instead. + +Layers: test_transformer_load_signature_supports_unsloth_kwargs (fast, runs when ST +is importable) and test_fast_sentence_transformer_matches_stock_st (end-to-end parity, +opt-in via UNSLOTH_EMBEDDING_PARITY_MODEL so default CI is unaffected). +""" + +from __future__ import annotations + +import inspect +import os + +import pytest + + +def test_transformer_load_signature_supports_unsloth_kwargs(): + """Forwards-compat tripwire: a Hub-capable Transformer.load must accept the kwargs + the #6881 fix passes. Legacy ST 3.x/4.x expose load(input_path); the code falls back + to Transformer(...) there, so mirror that gate and skip.""" + models = pytest.importorskip("sentence_transformers.models") + load = getattr(models.Transformer, "load", None) + assert callable(load), ( + "sentence_transformers Transformer.load is missing; the #6881 fix in " + "unsloth.models.sentence_transformer._create_transformer_module depends on it." + ) + params = inspect.signature(load).parameters + accepts_var_kw = any(p.kind is inspect.Parameter.VAR_KEYWORD for p in params.values()) + # Mirror _create_transformer_module's hub_capable gate. + hub_capable = accepts_var_kw or any(k in params for k in ("token", "cache_folder", "revision")) + if not hub_capable: + pytest.skip( + "legacy Transformer.load(input_path); production path falls back to Transformer(...)" + ) + unsupported = [ + k + for k in ("token", "cache_folder", "revision", "trust_remote_code") + if not (accepts_var_kw or k in params) + ] + assert not unsupported, ( + f"installed sentence_transformers Transformer.load no longer accepts {unsupported} " + f"and has no **kwargs; update _create_transformer_module (#6881) before it silently " + f"falls back to Transformer(...)." + ) + + +def _probe_texts(): + return [ + "roasted chickpeas in 20 kg bags", + "The capital of France is Paris.", + "A fast brown fox jumps over the lazy dog.", + "recette de tarte aux pommes traditionnelle", + ] + + +def test_fast_sentence_transformer_matches_stock_st(): + """End-to-end: FastSentenceTransformer embeddings and tokenization must match a + stock SentenceTransformer load of the same checkpoint. Opt-in (needs a model) and + GPU-only (FastSentenceTransformer requires CUDA), so it skips on CPU-only runners.""" + model_id = os.environ.get("UNSLOTH_EMBEDDING_PARITY_MODEL") + if not model_id: + pytest.skip( + "set UNSLOTH_EMBEDDING_PARITY_MODEL to a chat-template embedding model " + "(HF id or local path) to run the #6881 parity test" + ) + + torch = pytest.importorskip("torch") + if not torch.cuda.is_available(): + pytest.skip("FastSentenceTransformer requires CUDA; skipping on CPU-only runner") + np = pytest.importorskip("numpy") + pytest.importorskip("sentence_transformers") + from sentence_transformers import SentenceTransformer + + device = "cuda" + # Prefer bf16 when the GPU supports it: fp16 overflows to NaN on bf16-native + # embedders such as EmbeddingGemma (Gemma3), which would mask real parity. + dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16 + texts = _probe_texts() + max_seq_length = 256 + + # Control FIRST, before importing unsloth, so its global import patches never + # touch the stock reference (mirrors the issue's "restart runtime" repro). + ctrl = SentenceTransformer(model_id, device = device, model_kwargs = {"torch_dtype": dtype}) + ctrl.max_seq_length = max_seq_length + ctrl_ids = ctrl.tokenize([texts[0]])["input_ids"][0].tolist() + ctrl_emb = np.asarray( + ctrl.encode(texts, normalize_embeddings = True, batch_size = 8), dtype = np.float32 + ) + + import unsloth # noqa: F401 + from unsloth import FastSentenceTransformer + + fast = FastSentenceTransformer.from_pretrained( + model_id, + max_seq_length = max_seq_length, + dtype = dtype, + load_in_4bit = False, + load_in_16bit = True, + ) + fast_ids = fast.tokenize([texts[0]])["input_ids"][0].tolist() + fast_emb = np.asarray( + fast.encode(texts, normalize_embeddings = True, batch_size = 8), dtype = np.float32 + ) + + # Identical tokenization = no chat-template wrapping slipped in (the #6881 defect). + assert fast_ids == ctrl_ids, ( + f"tokenization diverged (chat-template wrapping regressed?):\n" + f" stock: {ctrl_ids}\n fast: {fast_ids}" + ) + + cos = (ctrl_emb * fast_emb).sum(1) / ( + np.linalg.norm(ctrl_emb, axis = 1) * np.linalg.norm(fast_emb, axis = 1) + ) + assert float(cos.min()) > 0.99, ( + f"embedding parity regressed: min cosine {float(cos.min()):.5f} <= 0.99 " + f"(per-text {[round(float(c), 5) for c in cos]})" + ) diff --git a/tests/version_compat/test_sentence_transformers_pinned_symbols.py b/tests/version_compat/test_sentence_transformers_pinned_symbols.py index c0c35b9d5d..d7f9a54811 100644 --- a/tests/version_compat/test_sentence_transformers_pinned_symbols.py +++ b/tests/version_compat/test_sentence_transformers_pinned_symbols.py @@ -19,6 +19,8 @@ ST_TAGS = [ "v5.2.3", "v5.3.0", "v5.4.1", + "v5.5.1", + "v5.6.0", "master", ] @@ -120,6 +122,42 @@ def test_st_transformer_base_class_either_path(tag: str): ) +# Transformer.load classmethod: unsloth builds saved-ST modules through it (#6881). +@pytest.mark.parametrize("tag", ST_TAGS) +def test_st_transformer_load_accepts_unsloth_kwargs(tag: str): + """unsloth builds saved ST models via Transformer.load(...) so the saved + modality_config is honored (#6881). If .load stops accepting the hub kwargs it + passes (and has no **kwargs), update the fix before it silently regresses. Not + locating .load is a SKIP (may be inherited); the live test guards the install.""" + candidates = [ + "sentence_transformers/models/Transformer.py", + "sentence_transformers/models/transformer.py", + "sentence_transformers/base/modules/transformer.py", + "sentence_transformers/base/modules/module.py", + ] + for p in candidates: + src = fetch_text("UKPLab/sentence-transformers", tag, p) + if src is None or not has_def(src, "load", "func"): + continue + m = re.search(r"def\s+load\s*\((.*?)\)\s*(?:->[^:]*)?:", src, re.S) + if m is None: + continue + sig = m.group(1) + accepts_var_kw = "**" in sig + missing = [ + kw + for kw in ("token", "cache_folder", "revision", "trust_remote_code") + if not (accepts_var_kw or re.search(rf"\b{re.escape(kw)}\b", sig)) + ] + assert not missing, ( + f"{tag}: Transformer.load in {p} no longer accepts {missing} and has no " + f"**kwargs; update unsloth.models.sentence_transformer._create_transformer_module " + f"(#6881) before it silently falls back to Transformer(...)." + ) + return + pytest.skip(f"{tag}: Transformer.load not locatable in {candidates} (may be inherited)") + + # sentence_transformers.util: import_from_string + load_dir_path helpers unsloth calls. @pytest.mark.parametrize("tag", ST_TAGS) def test_st_util_helpers(tag: str): diff --git a/unsloth/models/sentence_transformer.py b/unsloth/models/sentence_transformer.py index c1172faa94..4a1a555bcf 100644 --- a/unsloth/models/sentence_transformer.py +++ b/unsloth/models/sentence_transformer.py @@ -990,7 +990,17 @@ class FastSentenceTransformer(FastModel): return None @staticmethod - def _create_transformer_module(model_name, model, tokenizer, max_seq_length, trust_remote_code): + def _create_transformer_module( + model_name, + model, + tokenizer, + max_seq_length, + trust_remote_code, + token = None, + cache_dir = None, + revision = None, + module_subfolder = "", + ): """Helper to create and configure a Transformer module.""" from sentence_transformers.models import Transformer @@ -1077,7 +1087,45 @@ class FastSentenceTransformer(FastModel): elif "tokenizer_args" in transformer_init_params: transformer_kwargs["tokenizer_args"] = trust_remote_code_kwargs.copy() - transformer_module = Transformer(model_name, **transformer_kwargs) + # Build via Transformer.load so the saved modality_config is honored: plain + # Transformer(...) makes ST 5.x infer a "message" modality for chat-template + # models (e.g. Qwen3-Embedding), chat-wrapping inputs and degrading embeddings + # (#6881). Only use .load when it resolves a Hub id (accepts the kwargs or + # **kwargs); legacy ST 3.x/4.x load(input_path) is local-only with no modality + # bug, so fall back to the constructor. + transformer_module = None + transformer_load = getattr(Transformer, "load", None) + has_modules_json = ( + FastSentenceTransformer._module_path( + model_name, token, cache_dir = cache_dir, revision = revision + ) + is not None + ) + if callable(transformer_load) and has_modules_json: + load_params = inspect.signature(transformer_load).parameters + accepts_var_kw = any( + p.kind is inspect.Parameter.VAR_KEYWORD for p in load_params.values() + ) + hub_capable = accepts_var_kw or any( + key in load_params for key in ("token", "cache_folder", "revision") + ) + if hub_capable: + load_kwargs = { + "token": token, + "cache_folder": cache_dir, + "revision": revision, + "trust_remote_code": trust_remote_code, + **transformer_kwargs, + } + # Resolve config/tokenizer from the module's saved subfolder + # (modules.json "path"), like stock ST; "" (root) is a no-op. + if module_subfolder: + load_kwargs["subfolder"] = module_subfolder + if not accepts_var_kw: + load_kwargs = {k: v for k, v in load_kwargs.items() if k in load_params} + transformer_module = Transformer.load(model_name, **load_kwargs) + if transformer_module is None: + transformer_module = Transformer(model_name, **transformer_kwargs) finally: # Restore original Auto* loading immediately AutoModel.from_pretrained = original_model_from_pretrained @@ -1191,6 +1239,10 @@ class FastSentenceTransformer(FastModel): tokenizer, max_seq_length, trust_remote_code, + token, + cache_dir, + revision, + module_subfolder = module_config.get("path") or "", ) modules[name] = transformer_module else: @@ -1226,7 +1278,14 @@ class FastSentenceTransformer(FastModel): ) transformer_module = FastSentenceTransformer._create_transformer_module( - model_name, model, tokenizer, max_seq_length, trust_remote_code + model_name, + model, + tokenizer, + max_seq_length, + trust_remote_code, + token, + cache_dir, + revision, ) modules["0"] = transformer_module From 6d674e5cc9aef396ce8aae45306b2b42beb76244 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 9 Jul 2026 02:08:39 -0700 Subject: [PATCH 012/372] unsloth start: warn before running an agent's remote installer (#7024) When a coding agent is missing, `unsloth start ` offers to run the vendor's own installer (curl | bash, irm | iex, or npm) after an interactive confirm. Those installers execute with the user's privileges and there is no signature or hash check on the fetched content, so a blind "yes" is a supply-chain risk if the delivery path is compromised. Keep the auto-install convenience but make consent informed: before the prompt, name the exact remote source the installer fetches (or the command it runs for a package installer) and state that nothing verifies a signature or hash. Behavior is otherwise unchanged: non-interactive stdin still never executes anything, and the confirm still defaults to no. --- unsloth_cli/commands/start.py | 19 ++++++++++++++++++- unsloth_cli/tests/test_start.py | 26 ++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/unsloth_cli/commands/start.py b/unsloth_cli/commands/start.py index 764f5c7963..a8665b9be3 100644 --- a/unsloth_cli/commands/start.py +++ b/unsloth_cli/commands/start.py @@ -989,6 +989,12 @@ def _refresh_windows_path() -> None: os.environ["PATH"] = os.pathsep.join(entries) +def _install_source(install_hint: str) -> Optional[str]: + """The first http(s) URL an install hint fetches, or None (e.g. an npm install).""" + match = re.search(r"https?://[^\s'\")]+", install_hint) + return match.group(0) if match else None + + def _install_agent(name: str, install_hint: str) -> Optional[str]: # Missing agent under --launch: offer to run its documented install command, then # re-resolve it on PATH. Consent-based (we never auto-run a remote install script @@ -997,7 +1003,18 @@ def _install_agent(name: str, install_hint: str) -> Optional[str]: if not sys.stdin.isatty(): return None typer.echo(f"`{name}` is not installed.") - if not typer.confirm(f"Install it now with `{install_hint}`?", default = False): + # Make the supply-chain risk explicit before the prompt: these are the vendors' + # own installers (curl | bash, irm | iex, npm), run with the user's privileges, + # and nothing checks a signature or hash on the fetched content. Naming the source + # turns a blind "yes" into informed consent. + source = _install_source(install_hint) + warning = ( + f"This will download and RUN a script from {source} with your privileges" + if source + else f"This will RUN `{install_hint}` with your privileges" + ) + typer.secho(f"{warning}; there is no signature or hash check.", fg = "yellow", err = True) + if not typer.confirm(f"Install `{name}` now with `{install_hint}`?", default = False): return None # Run each hint through the shell it is written for: PowerShell (irm | iex, or npm) # on Windows, /bin/sh (curl | bash, or npm) everywhere else. diff --git a/unsloth_cli/tests/test_start.py b/unsloth_cli/tests/test_start.py index 18cb40f18d..87a295532e 100644 --- a/unsloth_cli/tests/test_start.py +++ b/unsloth_cli/tests/test_start.py @@ -128,6 +128,32 @@ def test_install_agent_uses_powershell_on_windows(monkeypatch): assert ran == [["powershell", "-NoProfile", "-Command", install_hint]] +def test_install_agent_warns_and_names_remote_source(monkeypatch, capsys): + # Before the confirm, a remote installer must name the URL it fetches so the + # user consents to a specific source rather than blindly accepting. + monkeypatch.setattr(start.os, "name", "nt") + monkeypatch.setattr(start.sys, "stdin", SimpleNamespace(isatty = lambda: True)) + monkeypatch.setattr(start.typer, "confirm", lambda *a, **k: False) # decline: nothing runs + hint = "& ([scriptblock]::Create((irm https://hermes-agent.nousresearch.com/install.ps1))) -SkipSetup" + assert start._install_agent("hermes", hint) is None + err = capsys.readouterr().err + assert "https://hermes-agent.nousresearch.com/install.ps1" in err + assert "download and RUN" in err + assert "signature or hash" in err + + +def test_install_agent_warns_for_package_installer(monkeypatch, capsys): + # An npm-style installer has no URL to fetch, but still runs with the user's + # privileges, so the warning names the command instead. + monkeypatch.setattr(start.os, "name", "posix") + monkeypatch.setattr(start.sys, "stdin", SimpleNamespace(isatty = lambda: True)) + monkeypatch.setattr(start.typer, "confirm", lambda *a, **k: False) + assert start._install_agent("codex", "npm install -g @openai/codex") is None + err = capsys.readouterr().err + assert "npm install -g @openai/codex" in err + assert "with your privileges" in err + + def test_hermes_install_hint_is_windows_native_on_windows(monkeypatch): monkeypatch.setattr(start.os, "name", "nt") From 0d4bd50768ca1d55009b51dfa097d7c71e819f4b Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 9 Jul 2026 02:26:24 -0700 Subject: [PATCH 013/372] Restore process-global torch.compile config on torch 2.12 so gradient checkpointing backward honors it (#7019) * Mirror dynamo/inductor config sets into defaults so torch 2.12 worker threads honor them torch 2.12 stores config user overrides in ContextVars, so direct assignments like torch._dynamo.config.recompile_limit = 1024 no longer reach the autograd engine worker threads. Gradient checkpointing recomputes fullgraph-compiled gpt-oss kernels inside backward on those threads, which then read the default recompile limit of 8 and raise FailOnRecompileLimitHit at step 0 of GRPO/SFT. Mirror direct config assignments into the process-global entry defaults on torch >= 2.12, restoring the torch <= 2.11 cross-thread semantics while leaving the context-scoped config.patch API untouched. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Keep config.patch thread-local when mirroring dynamo/inductor sets config.patch(...) also assigns through ConfigModule.__setattr__, so the default-mirror was leaking its scoped, thread-local writes into the process-global entry default. Track patch enter/exit with a per-thread depth counter (wrapping ConfigModule.patch) and skip mirroring while inside a patch, so only genuine direct assignments restore the torch 2.11 cross-thread semantics and config.patch stays context-local. * Also keep config.load_config thread-local when mirroring config sets load_config restores a saved dynamo/inductor config by calling setattr per key, which the default-mirror would otherwise leak process-wide just like config.patch did. Wrap load_config with the same per-thread depth counter (renamed to _scoped_depth) so both scoped writers skip the mirror and stay context-local, while genuine direct assignments still restore the torch 2.11 cross-thread default. * Drop the pre-existing override replay from the config thread fix The replay was redundant: this runs from _gpu_init before unsloth sets any dynamo/inductor config, so the __setattr__ wrapper already mirrors every later assignment (recompile_limit included). It could also read a value that belonged to a config.patch context still active at import time and write that thread-local override into the global default. Removing it keeps the cross-thread fix and drops the now-unused _inductor.config import. --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- unsloth/_gpu_init.py | 6 ++ unsloth/import_fixes.py | 132 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 136 insertions(+), 2 deletions(-) diff --git a/unsloth/_gpu_init.py b/unsloth/_gpu_init.py index e39b44488a..e6178e60f3 100644 --- a/unsloth/_gpu_init.py +++ b/unsloth/_gpu_init.py @@ -173,6 +173,7 @@ from .import_fixes import ( fix_vllm_guided_decoding_params, fix_vllm_pdl_blackwell, fix_triton_compiled_kernel_missing_attrs, + fix_dynamo_config_thread_visibility, patch_trunc_normal_precision_issue, ignore_logger_messages, patch_ipykernel_hf_xet, @@ -203,6 +204,10 @@ fix_vllm_guided_decoding_params() fix_trl_vllm_ascend() fix_vllm_pdl_blackwell() fix_triton_compiled_kernel_missing_attrs() +# Must run before unsloth_zoo's patch_torch_compile and the gpt-oss temporary +# patches raise the dynamo recompile limits, so those settings reach the +# autograd worker threads on torch >= 2.12. +fix_dynamo_config_thread_visibility() patch_trunc_normal_precision_issue() ignore_logger_messages() patch_ipykernel_hf_xet() @@ -233,6 +238,7 @@ del fix_vllm_guided_decoding_params del fix_trl_vllm_ascend del fix_vllm_pdl_blackwell del fix_triton_compiled_kernel_missing_attrs +del fix_dynamo_config_thread_visibility del patch_trunc_normal_precision_issue del ignore_logger_messages del patch_ipykernel_hf_xet diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index e5a5d01c2f..c5300ed4d0 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -1064,6 +1064,135 @@ def fix_triton_compiled_kernel_missing_attrs(): ) +def fix_dynamo_config_thread_visibility(): + """torch 2.12 made torch._dynamo/_inductor config overrides thread-local + (ContextVars), so `config.recompile_limit = 1024` set on the main thread is + invisible to the autograd worker threads that run backward. Gradient + checkpointing recompiles fullgraph gpt-oss kernels there against the default + limit of 8, raising FailOnRecompileLimitHit at step 0. Mirror direct config + assignments into the process-global entry default (torch <= 2.11 semantics). + config.patch(...) and config.load_config(...) also assign via __setattr__ but + are thread-local by design, so skip mirroring while inside one (tracked per + thread). No-op below torch 2.12 and on any torch without this internal layout. + """ + try: + import torch + + if Version(torch.__version__) < Version("2.12.0"): + return + import torch._dynamo.config as _dynamo_config + from torch.utils._config_module import ConfigModule + from contextvars import ContextVar + except Exception: + return + + try: + probe = getattr(_dynamo_config, "_config", {}).get("recompile_limit", None) + if probe is None or not isinstance(getattr(probe, "user_override", None), ContextVar): + # Overrides are not context-local on this torch; nothing to fix. + return + original_setattr = ConfigModule.__setattr__ + if getattr(original_setattr, "__unsloth_patched__", False): + return + except Exception: + return + + mirrored_modules = ("torch._dynamo.config", "torch._inductor.config") + + # config.patch(...) and config.load_config(...) also assign via __setattr__, but + # their writes are thread-local by design; a per-thread depth counter marks them + # so they are not mirrored into the process-global default. + import threading + + _scoped_depth = threading.local() + + def _in_scoped_write(): + return getattr(_scoped_depth, "n", 0) > 0 + + def _bump(delta): + _scoped_depth.n = getattr(_scoped_depth, "n", 0) + delta + + original_patch = ConfigModule.patch + if not getattr(original_patch, "__unsloth_patched__", False): + + @functools.wraps(original_patch) + def _patched_patch(self, *args, **kwargs): + ctx = original_patch(self, *args, **kwargs) + try: + cls = type(ctx) # patch() builds a fresh ConfigPatch class each call + if not getattr(cls, "__unsloth_patch_wrapped__", False): + _enter0, _exit0 = cls.__enter__, cls.__exit__ + + def _enter(s, _e = _enter0): + _bump(1) + try: + return _e(s) + finally: + _bump(-1) + + def _exit( + s, + *a, + _x = _exit0, + ): + _bump(1) + try: + return _x(s, *a) + finally: + _bump(-1) + + cls.__enter__, cls.__exit__ = _enter, _exit + cls.__unsloth_patch_wrapped__ = True + except Exception: + pass + return ctx + + _patched_patch.__unsloth_patched__ = True + ConfigModule.patch = _patched_patch + + # load_config restores a saved config by calling setattr per key (thread-local). + original_load_config = getattr(ConfigModule, "load_config", None) + if callable(original_load_config) and not getattr( + original_load_config, "__unsloth_patched__", False + ): + + @functools.wraps(original_load_config) + def _patched_load_config(self, *args, **kwargs): + _bump(1) + try: + return original_load_config(self, *args, **kwargs) + finally: + _bump(-1) + + _patched_load_config.__unsloth_patched__ = True + ConfigModule.load_config = _patched_load_config + + @functools.wraps(original_setattr) + def _patched_setattr(self, name, value): + original_setattr(self, name, value) + if _in_scoped_write(): + return # transient patch / load_config write: keep it thread-local + # Aliases (cache_size_limit -> recompile_limit) re-enter with the real name. + if self.__dict__.get("__name__", None) in mirrored_modules: + try: + entry = self.__dict__["_config"].get(name, None) + if entry is not None and entry.alias is None: + entry.default = value + except Exception: + pass + + _patched_setattr.__unsloth_patched__ = True + ConfigModule.__setattr__ = _patched_setattr + + # No replay of existing overrides: unsloth installs this before it sets any + # dynamo/inductor config, so the wrapper mirrors every later assignment. Replaying + # would also bake a still-active config.patch override into the global default. + logger.info( + "Unsloth: Patched torch config modules so dynamo/inductor settings " + "(e.g. recompile_limit) apply across threads on torch >= 2.12." + ) + + def patch_trunc_normal_precision_issue(): """ Patch torch.nn.init.trunc_normal_ for low precision tensors to run init in fp32. @@ -1323,8 +1452,7 @@ def fix_vllm_pdl_blackwell(): if patched: logger.info( - f"Unsloth: Applied PDL fix for SM100 ({sm100_gpu_name}) - " - f"patched: {', '.join(patched)}" + f"Unsloth: Applied PDL fix for SM100 ({sm100_gpu_name}) - patched: {', '.join(patched)}" ) else: # Just set the env var - vLLM might be an older version without supports_pdl From b509d47dd7427a1ba9ff1c80d1ca64fb9889bddf Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 9 Jul 2026 02:26:36 -0700 Subject: [PATCH 014/372] Silence torch._check_is_size FutureWarning and shim it if torch removes it (#7023) * Silence torch._check_is_size FutureWarning and shim it if torch removes it bitsandbytes 4-bit dequant calls torch._check_is_size, which torch deprecated with a FutureWarning ("Use _check(i >= 0) instead") that prints on every bnb-4bit load. Silence that warning in suppress_cuda_printf, and add fix_torch_check_is_size so a future torch that removes _check_is_size gets it shimmed to _check(i >= 0) (honoring the max bound) and bitsandbytes keeps working. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Tighten fix_torch_check_is_size docstring Lead with what the shim does and drop the redundant line; two lines instead of three, same intent. --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- unsloth/_gpu_init.py | 3 +++ unsloth/import_fixes.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/unsloth/_gpu_init.py b/unsloth/_gpu_init.py index e6178e60f3..984057e9f7 100644 --- a/unsloth/_gpu_init.py +++ b/unsloth/_gpu_init.py @@ -26,6 +26,7 @@ already_imported = [mod for mod in critical_modules if mod in sys.modules] # Fix some issues before importing other packages from .import_fixes import ( fix_message_factory_issue, + fix_torch_check_is_size, check_fbgemm_gpu_version, disable_broken_causal_conv1d, disable_broken_vllm, @@ -72,6 +73,7 @@ fix_bitsandbytes_rocm_arch_detection() disable_broken_causal_conv1d() disable_broken_vllm() fix_message_factory_issue() +fix_torch_check_is_size() check_fbgemm_gpu_version() torchvision_compatibility_check() fix_diffusers_warnings() @@ -81,6 +83,7 @@ del fix_bitsandbytes_rocm_arch_detection del disable_broken_causal_conv1d del disable_broken_vllm del fix_message_factory_issue +del fix_torch_check_is_size del check_fbgemm_gpu_version del torchvision_compatibility_check del fix_diffusers_warnings diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index c5300ed4d0..09de248c7b 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -172,6 +172,10 @@ if not UNSLOTH_ENABLE_LOGGING: # Deprecation warnings from torchao warnings.filterwarnings("ignore", message = "`int4_weight_only` is deprecated") warnings.filterwarnings("ignore", message = "`int8_weight_only` is deprecated") + # torch._check_is_size FutureWarning (called by bitsandbytes 4-bit dequant) + warnings.filterwarnings( + "ignore", message = r"_check_is_size will be removed", category = FutureWarning + ) # TorchAO deprecated import paths (https://github.com/pytorch/ao/issues/2752) warnings.filterwarnings( @@ -253,6 +257,30 @@ if not UNSLOTH_ENABLE_LOGGING: ) +def fix_torch_check_is_size(): + """Shim torch._check_is_size if a future torch removes it (bitsandbytes 4-bit + dequant calls it). The FutureWarning is silenced in suppress_cuda_printf.""" + try: + import torch + + if hasattr(torch, "_check_is_size"): + return + + def _check_is_size( + i, + message = None, + *, + max = None, + ): + torch._check(i >= 0, message) + if max is not None: + torch._check(i <= max, message) + + torch._check_is_size = _check_is_size + except Exception: + return + + # Fix up AttributeError: 'MessageFactory' object has no attribute 'GetPrototype' # MUST do this at the start primarily due to tensorflow causing issues def fix_message_factory_issue(): From c1e06e9ddfb53a9d40e1fb182030aded94f4b4bb Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 9 Jul 2026 02:47:59 -0700 Subject: [PATCH 015/372] unsloth start: add --persist to keep and reopen agent sessions (#7014) * unsloth start: add --resume to persist and reopen agent sessions `unsloth start ` launches a coding agent whose home is a throwaway temp dir wiped on exit, so codex/openclaw/hermes/pi (which relocate their whole home there) cannot resume a conversation after you quit. opencode and claude keep their session data in a fixed user dir, so they already resume. Add an opt-in --resume/--no-resume flag: it routes the launch to the stable Unsloth agents dir (the same one --no-launch already uses) so the session survives the exit, never touching the user's own ~/.. A bare --resume also reopens the last conversation via the agent's native flag (codex `resume --last`, opencode/claude/pi `--continue`). The default is unchanged: a plain launch still uses a temp dir and persists nothing. Add a dispatch-only `resume` job to the Local Agent Guides CI that drives the real launch path and asserts the split: codex/pi are wiped without --resume and persist with it, while opencode/claude persist either way. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * unsloth start: rename --resume to --persist The session flag collided with agents' own resume flags. `unsloth start claude --resume ` used to forward `--resume ` straight to Claude (which keeps its history in ~/.claude regardless), so a boolean --resume on unsloth start would have swallowed the session id and turned it into a stray prompt. Name the persistence flag --persist instead, so every agent's native resume flag (claude --resume , codex resume, opencode --continue, ...) still passes through untouched. Behavior is otherwise identical: --persist keeps a launched agent's session under the Unsloth agents dir, and a bare --persist reopens the last conversation. Add a regression test that `--resume ` passes through verbatim, and in the CI resume experiment skip the redundant second pass for opencode/claude (they persist either way, and a second CPU turn only risks a timeout). * unsloth start: correct --persist help and drop the buggy auto-resume Reword the --persist help to be accurate: claude and opencode keep sessions in the user's own stores and resume regardless, so --persist only stabilizes the otherwise-ephemeral relocated home of codex/openclaw/hermes/pi. Drop the bare-launch auto-append of native resume tokens: it errored on a first launch with no prior session, and was inconsistent between launch and no-launch. --persist now only keeps the session dir; resume via the agent's own command (e.g. `unsloth start codex --persist resume`), which now finds it. In the CI resume experiment, fail the pass when the launched turn exits non-zero, so a write-then-error is not misread as PERSISTED. --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .github/scripts/agent-guides-drive.sh | 148 +++++++++++++++++ .github/workflows/local-agent-guides-ci.yml | 170 ++++++++++++++++++++ unsloth_cli/commands/start.py | 53 ++++-- unsloth_cli/tests/test_start.py | 142 ++++++++++++++++ 4 files changed, 502 insertions(+), 11 deletions(-) diff --git a/.github/scripts/agent-guides-drive.sh b/.github/scripts/agent-guides-drive.sh index defdb498c7..f4189a159e 100755 --- a/.github/scripts/agent-guides-drive.sh +++ b/.github/scripts/agent-guides-drive.sh @@ -527,6 +527,154 @@ case "$MODE" in echo "[claude] attribution A/B OK (suppressed HIT, header=1 MISS)" ;; + # ── resume: does a launched agent's session survive exit and resume? ──── + # Unlike the other modes, this drives the real LAUNCH path (`unsloth start + # ...`, the interactive default), not the --no-launch recipe. That + # path relocates each agent's home to a throwaway temp dir wiped on exit, so + # a session cannot be resumed -- unless --persist routes it to the stable + # Unsloth agents dir instead. We run one headless turn per pass and check + # whether the turn left a session in a persistent store (deterministic, no + # reliance on the model recalling anything), for a baseline pass and a + # --persist pass, and assert the expected split for this agent. + resume) + CODEWORD="PLATYPUS7" + T1="Remember this codeword for later: ${CODEWORD}. Reply with just the word OK." + T2="What codeword did I ask you to remember? Reply with just that word." + WORK="$WORKDIR_BASE/${AGENT}-resume" + + # STABLE_HOME: the stable dir that --no-launch (and --persist) relocate to. + # Read it from a --no-launch probe (which also writes the agent's config + # there). codex/pi relocate their whole home/HOME here; opencode/claude keep + # their session data in a fixed user dir, so STABLE_HOME stays empty for them. + parse_connect + case "$AGENT" in + codex) STABLE_HOME="$(raw_env CODEX_HOME)" ;; + pi) STABLE_HOME="$(raw_env HOME)" ;; + *) STABLE_HOME="" ;; + esac + + # The persistent stores a session would land in if it were NOT wiped. We + # count files here before/after each turn; a positive delta means the + # session persisted (is resumable), zero means it went to a wiped temp dir. + resume_tracked_dirs() { + case "$AGENT" in + codex) printf '%s\n' "$HOME/.codex" ;; + opencode) printf '%s\n' "$HOME/.local/share/opencode" "$HOME/.config/opencode" ;; + claude) printf '%s\n' "$HOME/.claude" ;; + pi) printf '%s\n' "$HOME/.pi" ;; + *) : ;; + esac + [ -n "$STABLE_HOME" ] && printf '%s\n' "$STABLE_HOME" + } + count_session_files() { + local total=0 d n + while IFS= read -r d; do + [ -n "$d" ] && [ -d "$d" ] || continue + n="$(find "$d" -type f 2>/dev/null | wc -l)"; total=$((total + n)) + done < <(resume_tracked_dirs) + echo "$total" + } + + # The headless first-turn subcommand per agent (mirrors file-edit's map), + # forwarded verbatim through the launch path as passthrough args. + set_t1_cmd() { + case "$AGENT" in + claude) T1_CMD=("${CLAUDE_CONNECT_FLAGS[@]}" -p "$T1") ;; + codex) T1_CMD=(exec "$T1") ;; + opencode) T1_CMD=(run "$T1") ;; + pi) T1_CMD=(-p "$T1") ;; + *) guide_fail "resume mode does not cover agent '$AGENT'" ;; + esac + } + + # Run one headless turn through the launch path. $1=outfile, $2="" or + # "--persist", rest = the agent subcommand. --yolo auto-approves so no tool + # prompt can hang; --api-key attaches to the already-served CI model. + launch_turn() { + local out="$1" rflag="$2"; shift 2 + local flag=(); [ -n "$rflag" ] && flag=("$rflag") + run_timed "$out" unsloth start "$AGENT" "${flag[@]}" --yolo \ + --api-key "$UNSLOTH_API_KEY" "$@" + local rc=$? + redact "$out" + return "$rc" + } + + # One pass: fresh work dir, one planting turn, set RESULT to PERSISTED/WIPED + # from the session-store delta. Runs in the main shell (not a command + # substitution) so a hang's guide_fail actually fails the job and the + # progress lines reach the CI log. $1 = "" (baseline) or "--persist". + RESULT="" + run_pass() { + local rflag="$1" label="baseline" + [ -n "$rflag" ] && label="resume" + rm -rf "$WORK"; mkdir -p "$WORK" + set_t1_cmd + local out="$LOGS_DIR/${AGENT}-resume-${label}.txt" + local before after rc + before="$(count_session_files)" + pushd "$WORK" >/dev/null || guide_fail "could not enter work dir $WORK" + launch_turn "$out" "$rflag" "${T1_CMD[@]}"; rc=$? + popd >/dev/null || true + after="$(count_session_files)" + echo "[$AGENT] ${label}: session files ${before} -> ${after} (rc=${rc})" + # The turn must succeed for the delta to mean anything: an agent that writes a + # session file then errors would otherwise be misread as PERSISTED. Mirror the + # file-edit mode and fail the pass on a non-zero launch (the flagship codex recall + # below stays WARN-only, driven by its own launch_turn calls). + [ "$rc" -eq 0 ] || { echo "[$AGENT] ${label} transcript (tail):"; tail -30 "$out" 2>/dev/null || true; \ + guide_fail "resume ${label} turn for ${AGENT} exited non-zero (rc=${rc})"; } + if [ "$after" -gt "$before" ]; then RESULT="PERSISTED"; else RESULT="WIPED"; fi + } + + run_pass ""; BASELINE="$RESULT" + # Only the temp-dir agents (codex/pi) need the --persist pass to prove the fix. + # opencode/claude persist either way, so the baseline already proves it and a + # second full CPU turn only risks a timeout; skip it for them. + case "$AGENT" in + codex|pi) run_pass "--persist"; RESUME="$RESULT" ;; + *) RESUME="n/a (persists either way)" ;; + esac + + # Expected: codex/pi relocate their whole home to the temp dir, so a plain + # launch is WIPED and only --persist PERSISTS. opencode/claude keep their + # session data in a fixed user dir, so the baseline already PERSISTS. + case "$AGENT" in + codex|pi) EXPECT_BASELINE="WIPED" ;; + opencode|claude) EXPECT_BASELINE="PERSISTED" ;; + esac + + echo "──────────────────────────────────────────────" + echo "[$AGENT] RESUME EXPERIMENT" + echo " baseline (unsloth start ${AGENT}): ${BASELINE} (expected ${EXPECT_BASELINE})" + echo " with --persist (unsloth start ${AGENT} --persist): ${RESUME}" + echo "──────────────────────────────────────────────" + + [ "$BASELINE" = "$EXPECT_BASELINE" ] \ + || guide_fail "baseline resume behavior for ${AGENT} was ${BASELINE}, expected ${EXPECT_BASELINE}" + case "$AGENT" in + codex|pi) + [ "$RESUME" = "PERSISTED" ] \ + || guide_fail "--persist did not persist ${AGENT}'s session (got ${RESUME}); the session dir is still not stable" ;; + esac + + # Flagship behavioral proof (codex only, WARN-only): after a --persist plant, + # resume the session and check the model actually recalls the codeword. A + # miss is not a failure (the CI model is small); the mechanism gate above is + # the real assertion. + if [ "$AGENT" = "codex" ]; then + rm -rf "$WORK"; mkdir -p "$WORK" + ( cd "$WORK" && launch_turn "$LOGS_DIR/codex-resume-plant.txt" "--persist" exec "$T1" ) || true + ( cd "$WORK" && launch_turn "$LOGS_DIR/codex-resume-recall.txt" "--persist" exec resume --last "$T2" ) || true + if grep -q "$CODEWORD" "$LOGS_DIR/codex-resume-recall.txt" 2>/dev/null; then + echo "[codex] behavioral recall HIT: resumed session remembered ${CODEWORD}" + else + echo "::warning::[codex] behavioral recall MISS (small CI model); mechanism gate still passed" + fi + fi + echo "[$AGENT] resume OK" + ;; + *) echo "agent-guides-drive.sh: unknown mode '$MODE'" >&2 exit 2 diff --git a/.github/workflows/local-agent-guides-ci.yml b/.github/workflows/local-agent-guides-ci.yml index 47f75dc1ba..25796bd5cf 100644 --- a/.github/workflows/local-agent-guides-ci.yml +++ b/.github/workflows/local-agent-guides-ci.yml @@ -471,6 +471,176 @@ jobs: redacted-configs/ retention-days: 7 + # ═════════════════════════════════════════════════════════════════════ + # Job: resume + # Does a conversation started with `unsloth start ` survive exit + # and resume? This drives the REAL launch path (not the --no-launch + # recipe the other jobs use). A plain launch relocates the agent home to + # a temp dir wiped on exit, so codex/pi cannot resume; --persist routes the + # session to the stable Unsloth agents dir so it persists. opencode/claude + # keep their session data in a fixed user dir, so they persist either way. + # Dispatch-only: it is an end-to-end experiment, not a PR gate. + # ═════════════════════════════════════════════════════════════════════ + resume: + name: resume (${{ matrix.agent }}) + if: github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + # codex/pi relocate their whole home (resume broken without --persist); + # opencode/claude keep session data in a fixed dir (resume already works). + # One agent from each class proves the split end to end; openclaw/hermes + # share codex's relocation mechanism and are covered by the unit tests. + agent: [codex, opencode, claude, pi] + env: + GGUF_REPO: unsloth/gemma-4-E4B-it-GGUF + GGUF_FILE: gemma-4-E4B-it-UD-Q4_K_XL.gguf + STUDIO_PORT: '18904' + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Linux deps for llama.cpp prebuilt + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + libcurl4-openssl-dev libssl-dev jq + + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: '22' + + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.12' + cache: 'pip' + + - name: Restore GGUF model file + id: cache-gguf + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + continue-on-error: true + with: + path: gguf-cache + key: ${{ runner.os }}-gguf-${{ env.GGUF_REPO }}-${{ env.GGUF_FILE }}-v1 + + - name: Download GGUF if cache miss + id: download-gguf + if: steps.cache-gguf.outputs.cache-hit != 'true' || steps.cache-gguf.outcome != 'success' + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + run: | + python -m pip install --upgrade huggingface_hub + mkdir -p gguf-cache + bash .github/scripts/hf-download-with-retry.sh "$GGUF_REPO" "$GGUF_FILE" gguf-cache + + - name: Save GGUF model file + if: always() && steps.download-gguf.outcome == 'success' + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: gguf-cache + key: ${{ runner.os }}-gguf-${{ env.GGUF_REPO }}-${{ env.GGUF_FILE }}-v1 + + - name: Install Studio (--local, --no-torch) + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + HF_TOKEN: ${{ secrets.HF_TOKEN }} + run: | + mkdir -p logs + set -o pipefail + bash install.sh --local --no-torch 2>&1 | tee logs/install.log + + - name: Serve unsloth run --disable-tools (gemma-4-E4B) + run: | + unsloth studio reset-password + bash .github/scripts/serve-unsloth-run.sh \ + --gguf-file "$GITHUB_WORKSPACE/gguf-cache/${GGUF_FILE}" \ + --port "$STUDIO_PORT" --log-dir logs \ + --extra "--seed $UNSLOTH_SEED --temp 0" \ + --health-timeout 900 + + - name: Preflight the agent's API dialect (class-a isolation) + env: + AGENT: ${{ matrix.agent }} + run: | + set -uo pipefail + B="$UNSLOTH_BASE_URL"; K="$UNSLOTH_API_KEY" + preflight_fail() { + echo "::error::[server/API regression] agent=$AGENT: $* (preflight failed BEFORE install/connect). Endpoint contract lives in studio/backend/routes/**."; + exit 1 + } + code=$(curl -s -o /tmp/pf.json -w '%{http_code}' "$B/v1/models" \ + -H "Authorization: Bearer $K") || true + [ "$code" = "200" ] || preflight_fail "/v1/models returned HTTP $code" + case "$AGENT" in + claude) + code=$(curl -s -o /tmp/pf.json -w '%{http_code}' "$B/v1/messages" \ + -H "Authorization: Bearer $K" -H 'content-type: application/json' \ + --max-time 120 \ + -d "{\"model\":\"$UNSLOTH_MODEL_ID\",\"max_tokens\":16,\"messages\":[{\"role\":\"user\",\"content\":\"Hi\"}]}") || true + [ "$code" = "200" ] || preflight_fail "/v1/messages returned HTTP $code" + ;; + codex) + code=$(curl -s -o /tmp/pf.json -w '%{http_code}' "$B/v1/responses" \ + -H "Authorization: Bearer $K" -H 'content-type: application/json' \ + --max-time 120 \ + -d "{\"model\":\"$UNSLOTH_MODEL_ID\",\"input\":\"Hi\",\"max_output_tokens\":16,\"stream\":true}") || true + [ "$code" = "200" ] || preflight_fail "/v1/responses returned HTTP $code" + ;; + *) + code=$(curl -s -o /tmp/pf.json -w '%{http_code}' "$B/v1/chat/completions" \ + -H "Authorization: Bearer $K" -H 'content-type: application/json' \ + --max-time 120 \ + -d "{\"model\":\"$UNSLOTH_MODEL_ID\",\"max_tokens\":16,\"messages\":[{\"role\":\"user\",\"content\":\"Hi\"}]}") || true + [ "$code" = "200" ] || preflight_fail "/v1/chat/completions returned HTTP $code" + ;; + esac + echo "preflight OK for $AGENT" + + - name: Install agent CLI (class-b isolation) + env: + AGENT: ${{ matrix.agent }} + run: bash .github/scripts/agent-guides-install.sh "$AGENT" + + - name: Resume experiment (launch path) + env: + AGENT: ${{ matrix.agent }} + run: bash .github/scripts/agent-guides-drive.sh resume "$AGENT" + + - name: Collect server logs (debug) + if: always() + run: | + mkdir -p logs/studio-logs + cp -r "$HOME/.unsloth/studio/logs/." logs/studio-logs/ 2>/dev/null || true + if [ -n "${UNSLOTH_API_KEY:-}" ]; then + grep -rlF "$UNSLOTH_API_KEY" logs redacted-configs agent-workdir 2>/dev/null | while IFS= read -r f; do + sed -i "s#${UNSLOTH_API_KEY}##g" "$f" 2>/dev/null || true + done + fi + + - name: Stop Studio + if: always() + run: | + if [ -n "${UNSLOTH_SERVER_PID:-}" ] && [ "${UNSLOTH_SERVER_PID}" != "0" ]; then + kill "${UNSLOTH_SERVER_PID}" 2>/dev/null || true + fi + sleep 2 + ss -tln 2>/dev/null | grep ":${STUDIO_PORT}" || true + + - name: Upload logs + if: always() + continue-on-error: true + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: resume-${{ matrix.agent }}-log + path: | + logs/ + agent-workdir/ + redacted-configs/ + retention-days: 7 + # ═════════════════════════════════════════════════════════════════════ # Job 3: prompt-cache # (a) curl 2-turn /v1/chat/completions: assert turn-2 cached_tokens > 0 diff --git a/unsloth_cli/commands/start.py b/unsloth_cli/commands/start.py index a8665b9be3..48c0aca34b 100644 --- a/unsloth_cli/commands/start.py +++ b/unsloth_cli/commands/start.py @@ -133,6 +133,21 @@ _YOLO_OPTION = typer.Option( "flag/config. Any of the three spellings works for any agent." ), ) +_PERSIST_OPTION = typer.Option( + False, + "--persist/--no-persist", + help = ( + "Keep this agent's Unsloth-managed session dir so you can resume it later. " + "codex/openclaw/hermes/pi have their whole home relocated into an Unsloth dir " + "that is a throwaway temp dir (wiped on exit) by default; with --persist it " + "lives under the Unsloth agents dir and survives, so their own resume can reopen " + "it. claude and opencode keep sessions in your own stores (~/.claude, " + "~/.local/share/opencode), so they already resume regardless. To reopen a " + "session, pass the agent's own resume command through, e.g. " + "`unsloth start codex --persist resume` or `claude --resume `; those flow to " + "the agent unchanged." + ), +) # Per-agent CLI flag for "run tools without prompting". opencode and openclaw have no # such flag (config only) and are handled in their config writers, so they are absent. @@ -1133,15 +1148,20 @@ def _agents_config_root() -> Path: @contextlib.contextmanager -def _session_config(agent: str, launch: bool): +def _session_config( + agent: str, + launch: bool, + persist: bool = False, +): """Yield a private directory for an agent's session config (never the user's own). - launch: an ephemeral temp dir removed after the agent process exits, so nothing - persists. no-launch: a stable Unsloth-owned dir (the printed recipe is run later - on this machine), reused across runs. Either way the user's real ~/. - config is left untouched. + launch (default): an ephemeral temp dir removed after the agent process exits, so + nothing persists. no-launch: a stable Unsloth-owned dir (the printed recipe is run + later on this machine), reused across runs. persist (from --persist): use that same + stable dir even for a launch, so the agent's session survives the exit and can be + resumed next time. Either way the user's real ~/. config is left untouched. """ - if launch: + if launch and not persist: path = Path(tempfile.mkdtemp(prefix = f"unsloth-{agent}-")) try: yield path @@ -1453,6 +1473,7 @@ def claude( tensor_parallel: bool = _TENSOR_PARALLEL_OPTION, serve: bool = _SERVE_OPTION, yolo: bool = _YOLO_OPTION, + persist: bool = _PERSIST_OPTION, ): """Point Claude Code at the running Studio server and start it.""" base, key, entry = _connect( @@ -1497,6 +1518,9 @@ def claude( # --yolo (or its aliases) maps to Claude's own --dangerously-skip-permissions. # IS_SANDBOX is left unset on purpose: Claude refuses bypass mode as root unless a # sandbox is detected, and we don't want to falsely claim one on the user's host. + # claude keeps its history in ~/.claude/projects, which --settings/env never + # relocate, so a session already survives exit; resume it with `claude --continue` + # or `--resume ` passed through. command = [ "claude", "--model", @@ -1533,6 +1557,7 @@ def codex( tensor_parallel: bool = _TENSOR_PARALLEL_OPTION, serve: bool = _SERVE_OPTION, yolo: bool = _YOLO_OPTION, + persist: bool = _PERSIST_OPTION, ): """Point OpenAI Codex at the running Studio server and start it.""" base, key, entry = _connect( @@ -1558,7 +1583,7 @@ def codex( *_yolo_command_flags("codex", yolo), *ctx.args, ] - with _session_config("codex", launch) as home: + with _session_config("codex", launch, persist = persist) as home: write_codex_config(base, entry, home) env = {_CODEX_ENV_KEY: key, "CODEX_HOME": str(home)} _run(base, entry, env, command, launch = launch, install_hint = "npm install -g @openai/codex") @@ -1576,6 +1601,7 @@ def openclaw( tensor_parallel: bool = _TENSOR_PARALLEL_OPTION, serve: bool = _SERVE_OPTION, yolo: bool = _YOLO_OPTION, + persist: bool = _PERSIST_OPTION, ): """Point OpenClaw at the running Studio server and start it.""" base, key, entry = _connect( @@ -1601,7 +1627,7 @@ def openclaw( if os.name == "nt" else "curl -fsSL https://openclaw.ai/install.sh | bash" ) - with _session_config("openclaw", launch) as cfg: + with _session_config("openclaw", launch, persist = persist) as cfg: config_path = cfg / "openclaw.json" # key lives in the config, not the env; --yolo writes the exec policy here too. write_openclaw_config(base, key, entry, config_path, yolo = yolo) @@ -1622,6 +1648,7 @@ def opencode( tensor_parallel: bool = _TENSOR_PARALLEL_OPTION, serve: bool = _SERVE_OPTION, yolo: bool = _YOLO_OPTION, + persist: bool = _PERSIST_OPTION, ): """Point OpenCode at the running Studio server and start it.""" base, key, entry = _connect( @@ -1645,7 +1672,9 @@ def opencode( command = ["opencode", "--model", opencode_model] else: command = ["opencode"] - with _session_config("opencode", launch) as cfg: + # opencode keeps sessions in ~/.local/share/opencode (never relocated), so resume + # already survives exit; reopen the last one by passing `opencode --continue` through. + with _session_config("opencode", launch, persist = persist) as cfg: config_path = cfg / "opencode.json" # OPENCODE_CONFIG is an overlay (loaded between the user's global and project # configs), so this adds the Unsloth provider/model for the session without @@ -1697,6 +1726,7 @@ def hermes( tensor_parallel: bool = _TENSOR_PARALLEL_OPTION, serve: bool = _SERVE_OPTION, yolo: bool = _YOLO_OPTION, + persist: bool = _PERSIST_OPTION, ): """Point Hermes (Nous Research) at the running Studio server and start it.""" base, key, entry = _connect( @@ -1708,7 +1738,7 @@ def hermes( ) command = ["hermes", *_yolo_command_flags("hermes", yolo), *ctx.args] install_hint = _hermes_install_hint() - with _session_config("hermes", launch) as home: + with _session_config("hermes", launch, persist = persist) as home: # HERMES_HOME relocates hermes' whole home dir (config.yaml, sessions, state) # like CODEX_HOME, so the user's ~/.hermes is left untouched for the session. write_hermes_config(base, entry, home / "config.yaml") @@ -1728,6 +1758,7 @@ def pi( tensor_parallel: bool = _TENSOR_PARALLEL_OPTION, serve: bool = _SERVE_OPTION, yolo: bool = _YOLO_OPTION, + persist: bool = _PERSIST_OPTION, ): """Point Pi (coding agent) at the running Studio server and start it.""" base, key, entry = _connect( @@ -1752,7 +1783,7 @@ def pi( # --ignore-scripts matches Pi's documented install recipe (its README notes Pi needs # no install scripts), so accepting the prompt skips dependency lifecycle scripts. install_hint = "npm install -g --ignore-scripts @earendil-works/pi-coding-agent" - with _session_config("pi", launch) as home: + with _session_config("pi", launch, persist = persist) as home: # Pi resolves its config dir from PI_CODING_AGENT_DIR first (getAgentDir() prefers # it over $HOME/.pi/agent), so pin it at the session dir: an inherited # PI_CODING_AGENT_DIR in the user's shell would otherwise send Pi to their real diff --git a/unsloth_cli/tests/test_start.py b/unsloth_cli/tests/test_start.py index 87a295532e..065972b275 100644 --- a/unsloth_cli/tests/test_start.py +++ b/unsloth_cli/tests/test_start.py @@ -2548,3 +2548,145 @@ def test_session_config_no_launch_preserves_existing_state(fake_studio, tmp_path with start._session_config("codex", launch = False) as home2: assert home2 == home assert (home2 / "sessions" / "live.sqlite").read_text() == "state" + + +# ── --persist: persist the agent session so it can be resumed ──────────────── +def test_session_config_persist_uses_stable_dir_and_survives(monkeypatch, tmp_path): + # --persist routes a launch to the stable Unsloth agents dir (the one --no-launch + # already uses) instead of a throwaway temp dir, and never wipes it on exit. + monkeypatch.setattr(start, "_agents_config_root", lambda: tmp_path / "agents") + with start._session_config("codex", launch = True, persist = True) as home: + assert home == tmp_path / "agents" / "codex" + (home / "marker").write_text("kept") + assert home.exists() + assert (home / "marker").read_text() == "kept" + + +def test_session_config_default_launch_is_ephemeral(): + # Default launch (no --persist) still uses a throwaway temp dir wiped on exit. + with start._session_config("codex", launch = True) as home: + assert home.exists() + assert "unsloth-codex-" in home.name + assert not home.exists() + + +# The temp-dir agents: --persist points each one's home/state env at the stable dir; +# without it, at an ephemeral temp path. opencode is handled separately (only its +# config overlay is relocated; its session data was never in the temp dir). +_RESUME_ENV_VAR = { + "codex": "CODEX_HOME", + "openclaw": "OPENCLAW_STATE_DIR", + "hermes": "HERMES_HOME", + "pi": "HOME", +} + + +def _capture_launch(monkeypatch, argv): + captured = {} + + def run( + command, + env = None, + **kwargs, + ): + captured["command"] = command + captured["env"] = env + return SimpleNamespace(returncode = 0) + + monkeypatch.setattr(start.subprocess, "run", run) + result = CliRunner().invoke(start.start_app, argv) + assert result.exit_code == 0, result.output + return captured + + +@pytest.mark.parametrize("agent", sorted(_RESUME_ENV_VAR)) +def test_resume_persists_agent_home_to_stable_dir(agent, fake_studio, tmp_path, monkeypatch): + monkeypatch.setattr(start.shutil, "which", lambda _: f"/usr/local/bin/{agent}") + captured = _capture_launch(monkeypatch, [agent, "--persist"]) + stable = tmp_path / "agents" / agent + assert captured["env"][_RESUME_ENV_VAR[agent]] == str(stable) + # The stable dir survives the agent exit, so the session can be resumed. + assert stable.exists() + + +@pytest.mark.parametrize("agent", sorted(_RESUME_ENV_VAR)) +def test_default_launch_home_is_ephemeral(agent, fake_studio, tmp_path, monkeypatch): + monkeypatch.setattr(start.shutil, "which", lambda _: f"/usr/local/bin/{agent}") + captured = _capture_launch(monkeypatch, [agent]) + home = captured["env"][_RESUME_ENV_VAR[agent]] + assert f"unsloth-{agent}-" in home + assert str(tmp_path / "agents") not in home + + +def test_resume_opencode_config_in_stable_dir(fake_studio, tmp_path, monkeypatch): + # opencode's session data lives in ~/.local/share/opencode (never relocated), so + # resume already survives exit; --persist also stabilizes its config overlay dir. + monkeypatch.setattr(start.shutil, "which", lambda _: "/usr/local/bin/opencode") + captured = _capture_launch(monkeypatch, ["opencode", "--persist"]) + stable = tmp_path / "agents" / "opencode" + assert captured["env"]["OPENCODE_CONFIG"] == str(stable / "opencode.json") + assert stable.exists() + + +def test_persist_bare_codex_launch_has_no_resume_token(fake_studio, monkeypatch): + # A bare `--persist` only persists the session dir; it must NOT auto-append a native + # resume token, or the very first launch (no session yet) would send codex down its + # no-session error path. The user resumes explicitly: `unsloth start codex --persist resume`. + monkeypatch.setattr(start.shutil, "which", lambda _: "/usr/local/bin/codex") + captured = _capture_launch(monkeypatch, ["codex", "--persist"]) + assert "resume" not in captured["command"] + # command[0] is the resolved executable path; assert the argv after it. + assert captured["command"][1:] == ["--oss", "--profile", start._CODEX_PROFILE] + + +def test_persist_bare_opencode_launch_has_no_resume_token(fake_studio, monkeypatch): + monkeypatch.setattr(start.shutil, "which", lambda _: "/usr/local/bin/opencode") + captured = _capture_launch(monkeypatch, ["opencode", "--persist"]) + assert "--continue" not in captured["command"] + assert captured["command"][1:] == ["--model", f"{start._OPENCODE_PROVIDER}/{MODEL['id']}"] + + +def test_persist_bare_claude_launch_has_no_resume_token(fake_studio, monkeypatch): + monkeypatch.setattr(start.shutil, "which", lambda _: "/usr/local/bin/claude") + monkeypatch.setattr(start, "_claude_flags", lambda: []) + captured = _capture_launch(monkeypatch, ["claude", "--persist"]) + assert "--continue" not in captured["command"] + assert captured["command"][1:] == ["--model", MODEL["id"]] + + +def test_resume_with_passthrough_does_not_auto_append(fake_studio, monkeypatch): + # When the caller drives their own subcommand, --persist only persists the dir; it + # must not inject a resume token that would collide with the user's command. + monkeypatch.setattr(start.shutil, "which", lambda _: "/usr/local/bin/codex") + captured = _capture_launch(monkeypatch, ["codex", "--persist", "exec", "hello"]) + assert "resume" not in captured["command"] + assert captured["command"][-2:] == ["exec", "hello"] + + +def test_default_launch_has_no_resume_token(fake_studio, monkeypatch): + monkeypatch.setattr(start.shutil, "which", lambda _: "/usr/local/bin/codex") + captured = _capture_launch(monkeypatch, ["codex"]) + assert "resume" not in captured["command"] + + +def test_resume_persist_only_agents_have_no_resume_token(fake_studio, monkeypatch): + # openclaw/hermes persist their session dir but have no non-interactive resume + # selector, so --persist must not append a token; their own picker resumes. + for agent in ("openclaw", "hermes"): + monkeypatch.setattr(start.shutil, "which", lambda _, a = agent: f"/usr/local/bin/{a}") + captured = _capture_launch(monkeypatch, [agent, "--persist"]) + assert "resume" not in captured["command"] + assert "--continue" not in captured["command"] + + +def test_native_resume_flag_passes_through_unchanged(fake_studio, monkeypatch): + # The persistence flag is --persist, NOT --resume, so an agent's own + # `--resume ` (e.g. `unsloth start claude --resume `) still flows + # through to the agent verbatim and is not swallowed as a Studio option. + monkeypatch.setattr(start.shutil, "which", lambda _: "/usr/local/bin/claude") + monkeypatch.setattr(start, "_claude_flags", lambda: []) + captured = _capture_launch(monkeypatch, ["claude", "--resume", "some-session-guid"]) + assert captured["command"][-2:] == ["--resume", "some-session-guid"] + # Studio never auto-appends its own resume token when the user drives resume. + assert captured["command"].count("--resume") == 1 + assert "--continue" not in captured["command"] From eb775d320778bb496378344c061bd538e4d39ad9 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 9 Jul 2026 03:20:02 -0700 Subject: [PATCH 016/372] Studio /v1/messages: accept thinking and unknown content blocks (#7017) * Studio /v1/messages: accept thinking and unknown content blocks The Anthropic-compatible /v1/messages endpoint modeled a message's content as Union[str, list[{text|image|tool_use|tool_result}]], so any other block type made Pydantic reject the whole request with `messages.N.content.str: Input should be a valid string`. Resuming a Claude session commonly replays assistant turns that carry `thinking` (extended thinking) blocks, and sometimes a null content for a tool-only turn, both of which tripped this and returned a 400. Accept them: - Add a permissive AnthropicUnknownBlock fallback (any block whose type is not one of the four known ones), so thinking/redacted_thinking/provider-specific/ future blocks validate. A validator keeps known types on their typed models, so a malformed known block (e.g. a tool_use without id) still fails cleanly. - Coerce a null message (and tool_result) content to "" so the converter's `for block in content` stays safe. The converter already drops block types it does not translate, so a thinking block is not forwarded to the model. * Studio /v1/messages: keep user content validation strict Make the thinking/null leniency role-aware so it never silently drops real user input. Assistant turns (replayed history) still accept unknown/thinking blocks and coerce a null tool-only turn to empty. User turns keep the strict boundary: a null user content is rejected, and a content block the converter cannot translate is rejected instead of being dropped into an empty prompt. Also remove an empty file committed by accident. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Studio /v1/messages: coalesce resumed user turns and tighten content checks - The /v1/messages count and generation paths now coalesce the adjacent user turns that dropping an empty or null assistant turn can leave behind, so a strict GGUF chat template no longer 400s on non-alternating roles. - A user content block with a non-string type (list / dict) is rejected as a clean 400 instead of raising TypeError and escaping as a 500. - The assistant null-to-empty coercion only applies to an explicit null; an assistant turn that omits content entirely still fails required-field validation instead of being silently coerced to an empty string. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Studio /v1/messages: tighten comments --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- studio/backend/models/inference.py | 63 ++++++ studio/backend/routes/inference.py | 14 +- .../backend/tests/test_anthropic_messages.py | 184 ++++++++++++++++++ 3 files changed, 257 insertions(+), 4 deletions(-) diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py index 0f27b695fe..53b0f14b09 100644 --- a/studio/backend/models/inference.py +++ b/studio/backend/models/inference.py @@ -1533,12 +1533,41 @@ class AnthropicToolResultBlock(BaseModel): tool_use_id: str content: Union[str, list] = "" + @field_validator("content", mode = "before") + @classmethod + def _coerce_null_content(cls, v): + # Some clients send null content for an empty tool result; the str|list + # union would 400 on it, so treat null as "". + return "" if v is None else v + + +# Block types the converter translates explicitly. Anything else (thinking / +# redacted_thinking, a provider block a resumed session replays, or a future type) +# is accepted as an unknown block and dropped by the converter, rather than 400-ing +# the whole request on strict validation. +_KNOWN_ANTHROPIC_BLOCK_TYPES = frozenset({"text", "image", "tool_use", "tool_result"}) + + +class AnthropicUnknownBlock(BaseModel): + type: str + model_config = {"extra": "allow"} + + @field_validator("type") + @classmethod + def _only_unknown_types(cls, v): + # Known types parse as their typed models above (so a malformed known block + # still fails cleanly); this fallback only catches the rest. + if v in _KNOWN_ANTHROPIC_BLOCK_TYPES: + raise ValueError("known block type handled by its typed model") + return v + AnthropicContentBlock = Union[ AnthropicTextBlock, AnthropicImageBlock, AnthropicToolUseBlock, AnthropicToolResultBlock, + AnthropicUnknownBlock, ] @@ -1583,6 +1612,40 @@ class AnthropicMessage(BaseModel): role: Literal["user", "assistant"] content: Union[str, list[AnthropicContentBlock]] + @model_validator(mode = "before") + @classmethod + def _normalize_content(cls, data): + # Role-aware leniency that never silently drops real user input: + # - assistant: a resumed tool-only turn's null content -> "" (str|list would + # 400 on null; "" keeps the converter's `for block in content` safe). + # Unknown blocks (thinking / future types) validate via + # AnthropicUnknownBlock and are dropped by the converter. + # - user: keep strict. Null user content stays None so str|list rejects it + # (400) rather than forwarding an empty prompt; and reject block types the + # converter cannot translate, since it silently skips unknown user blocks + # -- a user turn made only of them would validate yet send no content + # (silent data loss). + if not isinstance(data, dict): + return data + content = data.get("content") + if data.get("role") == "assistant": + # Coerce only an explicit null (resumed tool-only turn). A missing + # content key stays malformed so the required-field check still 400s. + if "content" in data and content is None: + return {**data, "content": ""} + return data + if isinstance(content, list): + for block in content: + btype = ( + block.get("type") if isinstance(block, dict) else getattr(block, "type", None) + ) + # Guard the value: a non-string type is unsupported too, and a + # membership test on an unhashable value would raise TypeError + # (escaping as a 500 instead of a clean 400). + if not isinstance(btype, str) or btype not in _KNOWN_ANTHROPIC_BLOCK_TYPES: + raise ValueError(f"unsupported content block type {btype!r} in a user message") + return data + class AnthropicTool(BaseModel): # Client tools have input_schema; server tools may only have type/name. diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index d9901a5b2e..4bb9ce655e 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -10088,8 +10088,11 @@ async def anthropic_count_tokens( # Apply the same sanitization /messages does before generation, so the count # matches the prompt the real request would build (otherwise empty-assistant # sentinels / synthetic tool history inflate the count or hit the fallback). - openai_messages = _strip_provider_synthetic_tool_history( - _drop_empty_assistant_sentinels(openai_messages) + # Coalesce adjacent user turns left behind by dropping an empty / null assistant + # turn, so a strict GGUF chat template does not 400 on non-alternating roles + # (mirrors the GGUF chat path); a no-op for already-alternating histories. + openai_messages = _coalesce_consecutive_user_turns( + _strip_provider_synthetic_tool_history(_drop_empty_assistant_sentinels(openai_messages)) ) openai_tools = anthropic_tools_to_openai(payload.tools or []) or None @@ -10217,8 +10220,11 @@ async def anthropic_messages( # builders apply the same strip; without it an Anthropic /v1/messages caller # replaying a prior provider-side tool_use forwards fake builtin tool # history to a backend with no matching function declarations. - openai_messages = _strip_provider_synthetic_tool_history( - _drop_empty_assistant_sentinels(openai_messages) + # Coalesce adjacent user turns left behind by dropping an empty / null assistant + # turn, so a strict GGUF chat template does not 400 on non-alternating roles + # (mirrors the GGUF chat path); a no-op for already-alternating histories. + openai_messages = _coalesce_consecutive_user_turns( + _strip_provider_synthetic_tool_history(_drop_empty_assistant_sentinels(openai_messages)) ) # Enforce vision guard + re-encode embedded images to PNG so the Anthropic diff --git a/studio/backend/tests/test_anthropic_messages.py b/studio/backend/tests/test_anthropic_messages.py index 0c6550a3bb..54454f6563 100644 --- a/studio/backend/tests/test_anthropic_messages.py +++ b/studio/backend/tests/test_anthropic_messages.py @@ -1770,3 +1770,187 @@ class TestAnthropicMessagesToolRouting: _drive(anthropic_messages(payload, request = None, current_subject = "t")) assert backend.calls[0][0] == "plain" + + +def test_resumed_session_thinking_and_null_content_do_not_400(): + # A resumed session replays assistant turns with `thinking` (and sometimes null) + # content. Those must be accepted (thinking dropped by the converter), not 400ed. + from pydantic import ValidationError + + req = AnthropicMessagesRequest( + model = "x", + max_tokens = 16, + messages = [ + {"role": "user", "content": "hi"}, + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "secret reasoning", "signature": "s"}, + {"type": "text", "text": "the answer"}, + {"type": "tool_use", "id": "t1", "name": "f", "input": {}}, + ], + }, + {"role": "assistant", "content": None}, # tool-only turn serialized as null + ], + ) + # Known blocks still parse as their typed models; only the unknown one is loose. + assert type(req.messages[1].content[0]).__name__ == "AnthropicUnknownBlock" + assert type(req.messages[1].content[1]).__name__ == "AnthropicTextBlock" + assert req.messages[2].content == "" # null coerced + + openai = anthropic_messages_to_openai([m.model_dump() for m in req.messages]) + assistant = next(m for m in openai if m["role"] == "assistant" and m.get("content")) + assert assistant["content"] == "the answer" + assert "secret reasoning" not in json.dumps(openai) # thinking never forwarded + + # A malformed KNOWN block still fails cleanly instead of being swallowed. + with pytest.raises(ValidationError): + AnthropicMessagesRequest( + model = "x", + max_tokens = 16, + messages = [{"role": "assistant", "content": [{"type": "tool_use", "name": "f"}]}], + ) + + +def test_user_null_content_rejected(): + # The null->"" leniency is assistant-only; a null user content must be rejected + # at the boundary, not coerced into an empty prompt and forwarded to the model. + from pydantic import ValidationError + with pytest.raises(ValidationError): + AnthropicMessagesRequest( + model = "x", + max_tokens = 16, + messages = [{"role": "user", "content": None}], + ) + + +def test_user_unknown_block_rejected_not_silently_dropped(): + # The converter skips user blocks it cannot translate, so a user turn whose only + # block is unknown would validate yet forward no content. Reject at the boundary + # to avoid that silent data loss (the assistant fallback is unaffected). + from pydantic import ValidationError + with pytest.raises(ValidationError): + AnthropicMessagesRequest( + model = "x", + max_tokens = 16, + messages = [ + {"role": "user", "content": [{"type": "document", "source": {}}]}, + ], + ) + + +def test_user_translatable_blocks_still_accepted(): + # text / image / tool_result are translatable, so a real user message built from + # them must still pass; the unknown-block guard only trips on other types. + req = AnthropicMessagesRequest( + model = "x", + max_tokens = 16, + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is this?"}, + { + "type": "image", + "source": {"type": "base64", "media_type": "image/png", "data": "AA"}, + }, + {"type": "tool_result", "tool_use_id": "t1", "content": "ok"}, + ], + } + ], + ) + assert [type(b).__name__ for b in req.messages[0].content] == [ + "AnthropicTextBlock", + "AnthropicImageBlock", + "AnthropicToolResultBlock", + ] + + openai = anthropic_messages_to_openai([m.model_dump() for m in req.messages]) + assert any(m["role"] == "tool" and m["tool_call_id"] == "t1" for m in openai) + + +def test_user_malformed_known_block_still_rejected(): + # The guard only allow-lists a user block's *type*; the union still validates its + # shape, so a known-but-malformed block (tool_result without tool_use_id) fails. + from pydantic import ValidationError + with pytest.raises(ValidationError): + AnthropicMessagesRequest( + model = "x", + max_tokens = 16, + messages = [ + {"role": "user", "content": [{"type": "tool_result", "content": "x"}]}, + ], + ) + + +def test_user_content_block_non_string_type_rejected_cleanly(): + # A user block whose `type` is a non-string (unhashable list / dict, or a stray + # int) must fail as a clean validation error, not raise TypeError from the + # frozenset membership test and escape as a 500. + from pydantic import ValidationError + for bad_type in ([], {}, 5): + with pytest.raises(ValidationError): + AnthropicMessagesRequest( + model = "x", + max_tokens = 16, + messages = [{"role": "user", "content": [{"type": bad_type}]}], + ) + + +def test_assistant_missing_content_key_still_rejected(): + # The null -> "" leniency is only for an EXPLICIT null. An assistant message that + # omits content entirely stays malformed and must fail required-field validation. + from pydantic import ValidationError + + with pytest.raises(ValidationError): + AnthropicMessagesRequest( + model = "x", + max_tokens = 16, + messages = [{"role": "assistant"}], + ) + # An explicit null is still accepted and coerced (regression guard). + req = AnthropicMessagesRequest( + model = "x", + max_tokens = 16, + messages = [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": None}, + ], + ) + assert req.messages[1].content == "" + + +def test_resumed_null_assistant_between_users_coalesced_on_messages_route(monkeypatch): + # user -> assistant(null) -> user is now accepted: the null assistant turn coerces + # to "" and is dropped. The route must then coalesce the two remaining user turns + # so a strict GGUF chat template does not 400 on non-alternating roles. + backend = _mock_backend(monkeypatch, context_length = 2048) + + class _Req: + state = SimpleNamespace() + url = SimpleNamespace(path = "/v1/messages") + method = "POST" + + async def is_disconnected(self): + return False + + payload = AnthropicMessagesRequest( + model = "x", + max_tokens = 16, + messages = [ + {"role": "user", "content": "first question"}, + {"role": "assistant", "content": None}, + {"role": "user", "content": "please continue"}, + ], + ) + + response = _drive(anthropic_messages(payload, request = _Req(), current_subject = "t")) + assert response.status_code == 200 + + [(_path, kwargs)] = backend.calls + user_turns = [m for m in kwargs["messages"] if m.get("role") == "user"] + assert len(user_turns) == 1 # the two user turns were merged, not left adjacent + merged = user_turns[0]["content"] + if isinstance(merged, list): + merged = " ".join(p.get("text", "") for p in merged if isinstance(p, dict)) + assert "first question" in merged and "please continue" in merged From 350233512092fc6847b42050b7768f5f5e9a4578 Mon Sep 17 00:00:00 2001 From: oobabooga Date: Thu, 9 Jul 2026 07:39:48 -0300 Subject: [PATCH 017/372] Studio: add Vulkan llama.cpp support (#5819) * Studio: add Vulkan llama.cpp support * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Address gemini's feedback * Studio: move the Vulkan VRAM probe into a standalone script * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Improve Vulkan probe error reporting * Resolve llama-server symlink so Vulkan build is detected * Drop unreachable Vulkan fallback in GPU free-memory dispatcher * Skip the Intel GPU probe when NVIDIA or ROCm is present * Reserve host RAM headroom for Vulkan integrated GPUs * Add a `UNSLOTH_FORCE_VULKAN` environment variable * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Honor GGML_VK_VISIBLE_DEVICES, reserve discrete Vulkan VRAM headroom, and clear Intel GPU on --cpu-fallback * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Route Intel and forced-Vulkan hosts to the upstream Vulkan prebuilt, add arm64 Vulkan, keep Vulkan out of RAG auto-detect * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Clear the fork release pin when routing a Vulkan host to the upstream repo * Gate auto-Vulkan routing on no physical NVIDIA so hidden CUDA devices aren't used * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Pin Vulkan launches with --device Vulkan instead of the raw GGML_VK_VISIBLE_DEVICES index space * Let user --device override the Vulkan pin, and gate direct Vulkan asset picks on no physical NVIDIA * Update RAG auto-backend test mocks for the _resolve_auto binary and Vulkan probes * Keep the add_dll_directory handle alive through the Vulkan probe DLL loads * Revert RAG auto Vulkan guard, guard multi-backend Vulkan detection, and preserve forced Vulkan across updates * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Use getattr for RTLD_GLOBAL in the Vulkan probe CDLL mode * Skip CUDA/ROCm APU and datacenter GPU tuning on Vulkan builds On a Vulkan llama.cpp build gpu_indices are ggml compact ordinals, not CUDA/ROCm physical ids, so _amd_apu_wants_unified_memory and _apply_datacenter_env were reading the wrong device. On a mixed AMD APU plus discrete GPU host that could raise a spurious system-RAM shortfall and block a valid discrete-GPU load. Gate all three call sites on not is_vulkan_backend; the Vulkan path already reserves iGPU host headroom and the backend ignores GGML_CUDA_* anyway. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Tighten Vulkan-guard comment in load_model * Reduce comments in Vulkan support to be more succinct * Resolve shell-wrapper llama-server entrypoint to the real lib dir create_exec_entrypoint falls back to a #!/bin/sh wrapper at the install root when it cannot symlink into build/bin. _find_llama_server_binary returns that root entrypoint, but Path.resolve() does not follow a shell wrapper, so _llama_lib_dir returned the install root and _is_vulkan_backend missed libggml-vulkan.so -- silently skipping the Vulkan probe and --device pin on an otherwise valid Vulkan install. Follow the wrapper's exec target to build/bin. Regression test: test_shell_wrapper_entrypoint_resolves_to_real_lib_dir. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Lee Jackson <130007945+Imagineer99@users.noreply.github.com> Co-authored-by: danielhanchen --- .../backend/core/inference/_vulkan_probe.py | 110 ++++++++ studio/backend/core/inference/llama_cpp.py | 246 ++++++++++++++++-- .../tests/test_install_resolve_prebuilt.py | 170 ++++++++++++ studio/backend/tests/test_llama_cpp_update.py | 42 +++ .../tests/test_llama_cpp_vulkan_probe.py | 193 ++++++++++++++ studio/backend/utils/llama_cpp_update.py | 6 + studio/install_llama_prebuilt.py | 243 ++++++++++++++++- 7 files changed, 984 insertions(+), 26 deletions(-) create mode 100644 studio/backend/core/inference/_vulkan_probe.py create mode 100644 studio/backend/tests/test_llama_cpp_vulkan_probe.py diff --git a/studio/backend/core/inference/_vulkan_probe.py b/studio/backend/core/inference/_vulkan_probe.py new file mode 100644 index 0000000000..706346daad --- /dev/null +++ b/studio/backend/core/inference/_vulkan_probe.py @@ -0,0 +1,110 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Standalone free-VRAM probe for the bundled ggml Vulkan backend. + +Run in a short-lived subprocess (``python _vulkan_probe.py ``) so the +Vulkan instance never lives in the long-running backend process. Loads the +bundled ggml Vulkan backend from ```` and prints one +``\\t\\t\\t`` line per device to stdout. +Indices are ggml's own Vulkan device ordinals, which need not match nvidia-smi +order. ``is_igpu`` (from ggml's device type) is ``1`` for an integrated GPU +sharing system RAM. ``total_bytes`` is the device-local heap; the reader uses +it to reserve absolute headroom on a discrete card (parity with the CUDA/ROCm +fit) and ignores it for an iGPU, whose "VRAM" is shared system RAM. + +Uses only the standard library so it stays runnable as a bare script. +""" + +import ctypes +import os +import sys + +# ggml_backend_dev_type enum (ggml-backend.h): CPU=0, GPU=1, IGPU=2, ... +_GGML_BACKEND_DEVICE_TYPE_IGPU = 2 + + +def _igpu_flags(base, lib, count: int) -> list[bool]: + """Per-device integrated-GPU flags via ggml's backend registry. + + The Vulkan reg enumerates devices in the same order as + ``ggml_backend_vk_get_device_memory`` (each context uses ``ctx->device = + i``), so reg index == device ordinal. Returns all-False on any failure so + the reader never over-caps a discrete card. + """ + flags = [False] * count + try: + lib.ggml_backend_vk_reg.restype = ctypes.c_void_p + lib.ggml_backend_vk_reg.argtypes = [] + base.ggml_backend_reg_dev_count.restype = ctypes.c_size_t + base.ggml_backend_reg_dev_count.argtypes = [ctypes.c_void_p] + base.ggml_backend_reg_dev_get.restype = ctypes.c_void_p + base.ggml_backend_reg_dev_get.argtypes = [ctypes.c_void_p, ctypes.c_size_t] + base.ggml_backend_dev_type.restype = ctypes.c_int + base.ggml_backend_dev_type.argtypes = [ctypes.c_void_p] + + reg = lib.ggml_backend_vk_reg() + if not reg: + return flags + dev_count = base.ggml_backend_reg_dev_count(reg) + for i in range(min(count, dev_count)): + dev = base.ggml_backend_reg_dev_get(reg, i) + if dev: + flags[i] = base.ggml_backend_dev_type(dev) == _GGML_BACKEND_DEVICE_TYPE_IGPU + except Exception: + # Best-effort: any failure degrades to "discrete" so the memory + # readings still get through instead of crashing the probe. + pass + return flags + + +def main() -> int: + if len(sys.argv) < 2: + return 0 + bindir = sys.argv[1] + + # Hold add_dll_directory's handle for the rest of main() (the documented + # idiom) so bindir stays on the search path while the sibling ggml DLLs + # resolve below. + _dll_dir = None + if sys.platform == "win32": + base_name, vk_name = "ggml-base.dll", "ggml-vulkan.dll" + try: + _dll_dir = os.add_dll_directory(bindir) + except Exception: + pass + else: + base_name, vk_name = "libggml-base.so", "libggml-vulkan.so" + + # RTLD_GLOBAL exposes ggml-base's symbols to ggml-vulkan on POSIX. getattr + # falls back to 0 where the flag doesn't exist (Windows CDLL ignores mode). + _rtld_global = getattr(ctypes, "RTLD_GLOBAL", 0) + try: + base = ctypes.CDLL(os.path.join(bindir, base_name), mode = _rtld_global) + lib = ctypes.CDLL(os.path.join(bindir, vk_name), mode = _rtld_global) + except OSError as e: + print(f"ggml-vulkan load failed: {e}", file = sys.stderr) + return 1 + + lib.ggml_backend_vk_get_device_count.restype = ctypes.c_int + lib.ggml_backend_vk_get_device_count.argtypes = [] + lib.ggml_backend_vk_get_device_memory.restype = None + lib.ggml_backend_vk_get_device_memory.argtypes = [ + ctypes.c_int, + ctypes.POINTER(ctypes.c_size_t), + ctypes.POINTER(ctypes.c_size_t), + ] + + count = lib.ggml_backend_vk_get_device_count() + igpu = _igpu_flags(base, lib, count) + rows = [] + for i in range(count): + free, total = ctypes.c_size_t(0), ctypes.c_size_t(0) + lib.ggml_backend_vk_get_device_memory(i, ctypes.byref(free), ctypes.byref(total)) + rows.append("%d\t%d\t%d\t%d" % (i, free.value, int(igpu[i]), total.value)) + sys.stdout.write("\n".join(rows)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index f61402aa5c..3ba9eff857 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -1436,6 +1436,50 @@ def _backfill_usage_from_timings(usage, timings): return out +def _vulkan_lib_filename() -> str: + return "ggml-vulkan.dll" if sys.platform == "win32" else "libggml-vulkan.so" + + +# Host RAM to leave free on an integrated GPU, matching llama.cpp's own --fit +# margin (default 1024 MiB per device). ggml reports an iGPU's "VRAM" as shared +# system RAM, so hold back the same margin rather than inventing a larger one. +_IGPU_HOST_RESERVE_MIB = 1024 + + +def _apply_igpu_host_reserve_mib(free_mib: int, is_igpu: bool) -> int: + """Reserve host headroom on an integrated (shared-memory) Vulkan GPU. + + An iGPU's reported free "VRAM" is really free system RAM, so sizing + context/offload against all of it would push the host into swap or the OOM + killer. Leave the same margin llama.cpp's --fit uses. ``is_igpu`` comes from + ggml's device type, so a discrete card is never touched; only ever reduces. + """ + if not is_igpu: + return free_mib + return max(0, free_mib - _IGPU_HOST_RESERVE_MIB) + + +def _llama_lib_dir(binary: str) -> Path: + # The installer exposes llama-server as a top-level entrypoint into build/bin/, + # where the ggml backend libs live, so callers looking for sibling libs (Vulkan + # detection, LD_LIBRARY_PATH, probe bindir) need the real dir. It is normally a + # symlink (resolve() reaches build/bin), but create_exec_entrypoint falls back to + # a shell wrapper (exec "$(dirname "$0")/build/bin/llama-server" "$@") when it + # cannot symlink, and resolve() stops at the wrapper file. Follow the wrapper's + # exec target too, so a wrapper-based install still finds build/bin. + resolved = Path(binary).resolve() + try: + with open(resolved, "rb") as _f: + _head = _f.read(256) + if _head.startswith(b"#!"): + _m = re.search(r'exec "\$\(dirname "\$0"\)/([^"]+)"', _head.decode("utf-8", "ignore")) + if _m: + return (resolved.parent / _m.group(1)).resolve().parent + except OSError: + pass + return resolved.parent + + def _is_external_link(path: Path) -> bool: """True when ``path`` is a --with-llama-cpp-dir local link: a POSIX symlink or a Windows directory junction / reparse point. Such a link resolves into @@ -2278,6 +2322,30 @@ class LlamaCppBackend: return total + @staticmethod + def _is_vulkan_backend(binary: Optional[str] = None) -> bool: + """True if the installed llama.cpp build is Vulkan-only. + + The official prebuilts are single-backend, so the Vulkan ggml lib next + to llama-server identifies a Vulkan build. Keeps the free-memory probe + and GPU pin in ggml's Vulkan device-index space. For a custom + multi-backend build with a CUDA or HIP ggml lib alongside Vulkan, defer + to that backend (torch-usable, better-understood probe/pin). + """ + binary = binary or LlamaCppBackend._find_llama_server_binary() + if not binary: + return False + lib_dir = _llama_lib_dir(binary) + if not (lib_dir / _vulkan_lib_filename()).is_file(): + return False + for _backend in ("cuda", "hip"): + sibling = ( + f"ggml-{_backend}.dll" if sys.platform == "win32" else f"libggml-{_backend}.so" + ) + if (lib_dir / sibling).is_file(): + return False + return True + @staticmethod def _resolve_visible_physical_ids() -> Optional[list[int]]: """Physical GPU ids behind the active visibility mask (HIP/ROCR/CUDA on @@ -2440,11 +2508,42 @@ class LlamaCppBackend: return True @staticmethod - def _get_gpu_free_memory() -> list[tuple[int, int]]: + def _visible_devices_mask(env_name: str) -> Optional[set[int]]: + """Physical indices a ``*_VISIBLE_DEVICES`` mask permits, or None if unset. + + ``if x.strip()`` filters trailing-comma masks ("0,1,"); an empty mask + ("") yields an empty set (all devices hidden), distinct from an unset + var (None, no mask). Used by the nvidia-smi probe. + """ + raw = os.environ.get(env_name) + if raw is None: + return None + try: + return set(int(x.strip()) for x in raw.split(",") if x.strip()) + except ValueError: + return None + + @staticmethod + def _vulkan_pin_args(gpu_indices: Optional[Iterable[int]]) -> list[str]: + """``--device Vulkan,...`` to pin a Vulkan launch to selected GPUs. + + The indices are ggml's compact Vulkan ordinals (as _get_gpu_free_memory + reports and the registry names ``Vulkan``). Pin by that name, NOT via + GGML_VK_VISIBLE_DEVICES: ggml parses that env var in the raw + vkEnumeratePhysicalDevices space (before dropping CPU/llvmpipe devices + and deduplicating ICDs), so a compact ordinal there could select a + different physical device or the CPU rasterizer. + """ + if not gpu_indices: + return [] + return ["--device", ",".join(f"Vulkan{i}" for i in gpu_indices)] + + @staticmethod + def _get_gpu_free_memory(binary: Optional[str] = None) -> list[tuple[int, int]]: """Query free memory per GPU. Returns ``(gpu_index, free_mib)`` sorted by index; empty if no supported GPU is reachable. Thin wrapper over ``_get_gpu_memory`` for callers that only need free VRAM.""" - return [(idx, free) for idx, free, _total in LlamaCppBackend._get_gpu_memory()] + return [(idx, free) for idx, free, _total in LlamaCppBackend._get_gpu_memory(binary)] @staticmethod def _apple_metal_memory_budget_bytes() -> int: @@ -2475,7 +2574,7 @@ class LlamaCppBackend: return int(rec_bytes * _APPLE_UNIFIED_MEMORY_FRACTION) @staticmethod - def _get_gpu_memory() -> list[tuple[int, int, int]]: + def _get_gpu_memory(binary: Optional[str] = None) -> list[tuple[int, int, int]]: """Query free AND total memory per GPU. Order: @@ -2487,9 +2586,18 @@ class LlamaCppBackend: probe returned [] on AMD) and NVIDIA hosts missing ``nvidia-smi`` from PATH. + On a Vulkan build the ggml Vulkan probe is authoritative, so the indices + are ggml's compact Vulkan ordinals (the space the pin selects via + ``--device Vulkan``). It reports ``total`` for discrete cards and 0 + for an iGPU (shared RAM) so the fit falls back to free*frac there. + Otherwise nvidia-smi / torch cover NVIDIA + AMD ROCm. + Returns (gpu_index, free_mib, total_mib) sorted by index; empty if no - supported GPU is reachable. ``total`` lets the fit reserve absolute headroom. + supported GPU is reachable. """ + binary = binary or LlamaCppBackend._find_llama_server_binary() + if LlamaCppBackend._is_vulkan_backend(binary): + return LlamaCppBackend._get_gpu_free_memory_vulkan(binary) # ── NVIDIA via nvidia-smi ──────────────────────────────────── try: result = subprocess.run( @@ -2505,16 +2613,7 @@ class LlamaCppBackend: **_windows_hidden_subprocess_kwargs(), ) if result.returncode == 0: - allowed: Optional[set[int]] = None - cvd = os.environ.get("CUDA_VISIBLE_DEVICES") - if cvd is not None: - try: - # `if x.strip()` filters trailing-comma masks ("0,1,"). - # Empty mask (CVD="") yields an empty set -> all GPUs - # filtered out, per codebase convention. - allowed = set(int(x.strip()) for x in cvd.split(",") if x.strip()) - except ValueError: - pass + allowed = LlamaCppBackend._visible_devices_mask("CUDA_VISIBLE_DEVICES") gpus: list[tuple[int, int, int]] = [] for line in result.stdout.strip().splitlines(): parts = [p.strip() for p in line.split(",")] @@ -2579,6 +2678,91 @@ class LlamaCppBackend: logger.debug(f"torch GPU probe failed: {e}") return [] + @staticmethod + def _get_gpu_free_memory_vulkan(binary: Optional[str] = None) -> list[tuple[int, int, int]]: + """Query free (and total) VRAM per device via the bundled ggml Vulkan backend. + + Loads ``libggml-vulkan`` in a short-lived subprocess (no Vulkan instance + in this process) and returns (device_index, free_mib, total_mib) sorted + by index. The index is ggml's compact Vulkan ordinal -- the one the + registry names ``Vulkan`` and load_model pins with ``--device``, + NOT the raw ``GGML_VK_VISIBLE_DEVICES`` space. A user-set + ``GGML_VK_VISIBLE_DEVICES`` is honored by ggml (passed through), so the + list already reflects it. iGPUs leave a host-RAM margin (see + ``_apply_igpu_host_reserve_mib``) and report total 0; discrete cards pass + their real total through. [] when no Vulkan build or device is reachable. + """ + binary = binary or LlamaCppBackend._find_llama_server_binary() + if not binary: + return [] + binary_dir = _llama_lib_dir(binary) + if not (binary_dir / _vulkan_lib_filename()).is_file(): + return [] + + env = child_env_without_native_path_secret() + # Pass any inherited GGML_VK_VISIBLE_DEVICES through to ggml unchanged so + # the probe enumerates the same device list the launch will, named + # Vulkan0..N in the compact order reported here and pinned by that name + # via --device -- probe, mask, and pin stay in one index space. Do NOT + # filter the mask in Python: ggml parses the env var in raw + # vkEnumeratePhysicalDevices space while this probe reports the compact + # post-filter ordinal, so a Python filter would compare mismatched spaces. + if sys.platform != "win32": + # Let the loader resolve sibling ggml libs next to the binary. + existing_ld = env.get("LD_LIBRARY_PATH", "") + env["LD_LIBRARY_PATH"] = ( + f"{binary_dir}:{existing_ld}" if existing_ld else str(binary_dir) + ) + probe_script = Path(__file__).with_name("_vulkan_probe.py") + try: + result = subprocess.run( + [sys.executable, str(probe_script), str(binary_dir)], + capture_output = True, + text = True, + timeout = 15, + env = env, + **_windows_hidden_subprocess_kwargs(), + ) + if result.returncode != 0: + logger.debug( + f"vulkan GPU probe exited {result.returncode}: {result.stderr.strip()}" + ) + return [] + except Exception as e: + logger.debug(f"vulkan GPU probe failed: {e}") + return [] + + gpus: list[tuple[int, int, int]] = [] + for line in result.stdout.strip().splitlines(): + parts = line.split("\t") + if len(parts) != 4: + continue + try: + idx = int(parts[0]) + free_mib = int(parts[1]) // (1024 * 1024) + is_igpu = parts[2] == "1" + # iGPU "total" is shared RAM, not a VRAM budget -> keep 0 so the + # fit stays on free*frac (the host reserve below is its + # headroom); a discrete card passes its real total through. + total_mib = 0 if is_igpu else int(parts[3]) // (1024 * 1024) + except ValueError: + continue + capped = _apply_igpu_host_reserve_mib(free_mib, is_igpu) + if capped < free_mib: + logger.info( + f"Vulkan device VK{idx} is an integrated GPU sharing system " + f"RAM; reserving {free_mib - capped}MiB host headroom " + f"({free_mib}->{capped}MiB usable)" + ) + gpus.append((idx, capped, total_mib)) + gpus.sort(key = lambda g: g[0]) + if gpus: + logger.info( + "Vulkan GPU memory detected: " + + ", ".join(f"VK{idx}={free}MiB" for idx, free, _total in gpus) + ) + return gpus + @staticmethod def _available_system_memory_mib() -> Optional[int]: """Available system RAM in MiB (psutil, then /proc/meminfo), or None if @@ -2807,7 +2991,8 @@ class LlamaCppBackend: def _llama_server_env_for_binary(binary: str) -> dict[str, str]: """Build a subprocess env that lets llama-server resolve native libs.""" env = child_env_without_native_path_secret() - binary_dir = str(Path(binary).parent) + # _llama_lib_dir resolves the llama-server symlink to the real build/bin. + binary_dir = str(_llama_lib_dir(binary)) if sys.platform == "win32": # Ordering: see _build_windows_path_dirs. #5106. @@ -5210,6 +5395,7 @@ class LlamaCppBackend: # Resolve llama-server now but defer a not-found error: a block-diffusion # GGUF uses the diffusion runner, and its arch is only known after the header. binary = self._find_llama_server_binary() + is_vulkan_backend = self._is_vulkan_backend(binary) # ── Phase 2: download (NO lock held, so cancel can proceed) ── # mtp_draft_path arrives set for local Gemma loads (detected @@ -5449,7 +5635,8 @@ class LlamaCppBackend: model_size = gguf_size + mmproj_size # 2-tuple gpus for existing logic + a total map for the absolute # per-GPU headroom (correct when the GPU is already partly used). - _gpu_mem = self._get_gpu_memory() + # Pass binary so a Vulkan build probes ggml's Vulkan ordinals. + _gpu_mem = self._get_gpu_memory(binary) gpus = [(idx, free) for idx, free, _t in _gpu_mem] total_by_idx = {idx: total for idx, _f, total in _gpu_mem} @@ -6222,7 +6409,12 @@ class LlamaCppBackend: # cap, not the ROCm-reported VRAM, is the real ceiling); refuse an # oversize load the OS would otherwise kill mid-flight. Base model # only: an optional MTP drafter is dropped by the MTP-drop fallback. - if model_size is not None and self._amd_apu_wants_unified_memory(gpu_indices): + # CUDA/ROCm ids only; a Vulkan build's gpu_indices are ggml ordinals. + if ( + model_size is not None + and not is_vulkan_backend + and self._amd_apu_wants_unified_memory(gpu_indices) + ): _ram_msg = self._apu_ram_shortfall_message( model_size, self._available_system_memory_mib() ) @@ -6485,6 +6677,12 @@ class LlamaCppBackend: ", ".join(unsupported_cache_flags), ) + # Vulkan pins via --device (a cmd arg, unlike the env-based + # CUDA/ROCm pin below), emitted BEFORE user extras so llama.cpp's + # last-wins parsing lets a user --device override Studio's pick. + if is_vulkan_backend and gpu_indices is not None: + cmd += LlamaCppBackend._vulkan_pin_args(gpu_indices) + # User pass-through args go last so llama.cpp's last-wins parsing # lets the user override Studio's auto-set flags. Already # validated by the route via validate_extra_args(). @@ -6536,23 +6734,25 @@ class LlamaCppBackend: env.setdefault("OMP_NUM_THREADS", "2") # AMD unified-memory APUs (gfx1150/gfx1151): let llama.cpp use - # shared system RAM. setdefault so a user value wins. - if self._amd_apu_wants_unified_memory(gpu_indices): + # shared system RAM. setdefault so a user value wins. Not on Vulkan + # (nor DC below): gpu_indices are ggml ordinals, not CUDA/ROCm ids. + if not is_vulkan_backend and self._amd_apu_wants_unified_memory(gpu_indices): env.setdefault("GGML_CUDA_ENABLE_UNIFIED_MEMORY", "1") logger.info("AMD unified-memory APU: set GGML_CUDA_ENABLE_UNIFIED_MEMORY=1") # DC NVIDIA GPUs: FP32 accum (+ P2P / launch queues for multi-GPU). # See _apply_datacenter_env; opt out with UNSLOTH_DISABLE_DC_TUNING=1. - if self._apply_datacenter_env(env, gpu_indices): + if not is_vulkan_backend and self._apply_datacenter_env(env, gpu_indices): multi_gpu = self._effective_gpu_count(gpu_indices) > 1 logger.info( f"Data-center GPU detected: applied DC llama.cpp env tuning (multi_gpu={multi_gpu})" ) # Pin to selected GPU(s). On ROCm, narrowing only - # CUDA_VISIBLE_DEVICES leaves an AMD child seeing the full - # set, so set HIP_VISIBLE_DEVICES too. - if gpu_indices is not None: + # CUDA_VISIBLE_DEVICES leaves an AMD child seeing the full set, so + # set HIP_VISIBLE_DEVICES too. Vulkan is pinned via --device + # (above), not here. + if gpu_indices is not None and not is_vulkan_backend: pinned = ",".join(str(i) for i in gpu_indices) env["CUDA_VISIBLE_DEVICES"] = pinned try: diff --git a/studio/backend/tests/test_install_resolve_prebuilt.py b/studio/backend/tests/test_install_resolve_prebuilt.py index b825172a63..090d2932ea 100644 --- a/studio/backend/tests/test_install_resolve_prebuilt.py +++ b/studio/backend/tests/test_install_resolve_prebuilt.py @@ -55,6 +55,27 @@ def _host(**kw): return ilp.HostInfo(**base) +def test_force_cpu_clears_all_gpu_attributes_including_intel(): + # --cpu-fallback is the "select the CPU prebuilt even when a GPU is present" + # escape hatch. It must drop EVERY GPU attribute, including has_intel_gpu, or + # the planner still prepends the Vulkan asset on an Intel-GPU host. + host = _host( + is_linux = True, + is_x86_64 = True, + has_usable_nvidia = True, + has_physical_nvidia = True, + has_rocm = True, + rocm_gfx_target = "gfx1100", + has_intel_gpu = True, + ) + forced = ilp._apply_host_overrides(host, force_cpu = True) + assert forced.has_usable_nvidia is False + assert forced.has_physical_nvidia is False + assert forced.has_rocm is False + assert forced.rocm_gfx_target is None + assert forced.has_intel_gpu is False + + def test_macos_upstream_pin_only_for_explicit_pre26_upstream(): pre26 = _host( system = "Darwin", @@ -313,3 +334,152 @@ def test_sm103_host_drops_cuda128_windows_build(): ) kept_b200 = ilp._drop_blackwell_incapable_windows_cuda(b200, [cuda128, cuda129]) assert [a.name for a in kept_b200] == [cuda128.name, cuda129.name] + + +def _upstream_release(tag, asset_names): + return { + "tag_name": tag, + "assets": [ + {"name": n, "browser_download_url": f"https://example/{n}"} for n in asset_names + ], + } + + +def test_direct_upstream_arm64_intel_prefers_vulkan(): + # Auto-detected Intel GPU on Linux arm64 -> Vulkan prebuilt first, CPU + # second (mirrors the x86_64 branch; ggml-org ships the arm64 Vulkan asset). + host = _host(is_linux = True, is_arm64 = True, machine = "aarch64", has_intel_gpu = True) + rel = _upstream_release( + "b9925", + ["llama-b9925-bin-ubuntu-vulkan-arm64.tar.gz", "llama-b9925-bin-ubuntu-arm64.tar.gz"], + ) + plan = ilp.direct_upstream_release_plan(rel, host, UPSTREAM, "latest") + kinds = [a.install_kind for a in plan.attempts] + assert kinds[0] == "linux-vulkan", kinds + assert "linux-arm64" in kinds + assert plan.attempts[0].name == "llama-b9925-bin-ubuntu-vulkan-arm64.tar.gz" + + +def test_direct_upstream_intel_with_hidden_nvidia_is_cpu_only(): + # A host with a physical NVIDIA hidden via CUDA_VISIBLE_DEVICES (physical + # True, usable False) + an Intel iGPU must NOT get the Vulkan archive even + # when planning directly against upstream: Vulkan ignores CUDA_VISIBLE_DEVICES + # and could grab the reserved card. It falls through to the CPU asset. + host = _host( + is_linux = True, + is_x86_64 = True, + has_intel_gpu = True, + has_physical_nvidia = True, + has_usable_nvidia = False, + ) + rel = _upstream_release( + "b9925", + ["llama-b9925-bin-ubuntu-vulkan-x64.tar.gz", "llama-b9925-bin-ubuntu-x64.tar.gz"], + ) + plan = ilp.direct_upstream_release_plan(rel, host, UPSTREAM, "latest") + assert [a.install_kind for a in plan.attempts] == ["linux-cpu"] + + +def test_direct_upstream_arm64_without_intel_is_cpu_only(): + host = _host(is_linux = True, is_arm64 = True, machine = "aarch64") + rel = _upstream_release( + "b9925", + ["llama-b9925-bin-ubuntu-vulkan-arm64.tar.gz", "llama-b9925-bin-ubuntu-arm64.tar.gz"], + ) + plan = ilp.direct_upstream_release_plan(rel, host, UPSTREAM, "latest") + assert [a.install_kind for a in plan.attempts] == ["linux-arm64"] + + +def test_direct_upstream_x86_intel_prefers_vulkan(): + host = _host(is_linux = True, is_x86_64 = True, has_intel_gpu = True) + rel = _upstream_release( + "b9925", + ["llama-b9925-bin-ubuntu-vulkan-x64.tar.gz", "llama-b9925-bin-ubuntu-x64.tar.gz"], + ) + plan = ilp.direct_upstream_release_plan(rel, host, UPSTREAM, "latest") + kinds = [a.install_kind for a in plan.attempts] + assert kinds[0] == "linux-vulkan", kinds + assert "linux-cpu" in kinds + + +def test_linux_vulkan_health_glob_matches_bare_cpu_lib(): + # The widened glob must cover both arch-suffixed (x64) and bare (arm64) CPU + # libs so a valid Vulkan install is not re-flagged unhealthy every check. + choice = ilp.AssetChoice( + repo = UPSTREAM, + tag = "b9925", + name = "llama-b9925-bin-ubuntu-vulkan-arm64.tar.gz", + url = "https://example/x", + source_label = "upstream", + install_kind = "linux-vulkan", + ) + groups = ilp.runtime_payload_health_groups(choice) + assert ["libggml-cpu*.so*"] in groups + assert ["libggml-cpu-*.so*"] not in groups + + +def test_route_to_vulkan_prebuilt_auto_intel_goes_upstream_and_drops_fork_pin(): + # Routing fork -> upstream also drops the fork release pin, which is in a + # different tag namespace and would make the upstream resolver miss. + host = _host(is_linux = True, is_x86_64 = True, has_intel_gpu = True) + routed, repo, tag = ilp._route_to_vulkan_prebuilt(host, FORK, "b9596-mix-abc", force_cpu = False) + assert repo == UPSTREAM + assert tag == "" + assert routed.has_intel_gpu is True + + +def test_route_to_vulkan_prebuilt_preserves_explicit_upstream_pin(): + # A pin set WITH an explicit upstream repo is already on upstream -> kept. + host = _host(is_linux = True, is_x86_64 = True, has_intel_gpu = True) + _routed, repo, tag = ilp._route_to_vulkan_prebuilt(host, UPSTREAM, "b9596", force_cpu = False) + assert repo == UPSTREAM + assert tag == "b9596" + + +def test_route_to_vulkan_prebuilt_cpu_fallback_wins(): + # --cpu-fallback suppresses Vulkan routing even for an Intel host. + host = _host(is_linux = True, is_x86_64 = True, has_intel_gpu = True) + routed, repo, tag = ilp._route_to_vulkan_prebuilt(host, FORK, "b9596-mix-abc", force_cpu = True) + assert repo == FORK + assert tag == "b9596-mix-abc" + assert routed is host + + +def test_route_to_vulkan_prebuilt_hidden_nvidia_not_rerouted(): + # A mixed NVIDIA+Intel host that hid NVIDIA (CUDA_VISIBLE_DEVICES=""/-1): + # physical NVIDIA present but not usable. Must NOT auto-route to Vulkan, or + # Vulkan (which ignores CUDA_VISIBLE_DEVICES) could grab the reserved GPU. + host = _host( + is_linux = True, + is_x86_64 = True, + has_intel_gpu = True, + has_physical_nvidia = True, + has_usable_nvidia = False, + ) + _routed, repo, _tag = ilp._route_to_vulkan_prebuilt(host, FORK, "", force_cpu = False) + assert repo == FORK + + +def test_route_to_vulkan_prebuilt_rocm_host_not_rerouted(): + # An Intel iGPU alongside a usable ROCm GPU stays on its ROCm/fork path. + host = _host(is_linux = True, is_x86_64 = True, has_intel_gpu = True, has_rocm = True) + _routed, repo, _tag = ilp._route_to_vulkan_prebuilt(host, FORK, "", force_cpu = False) + assert repo == FORK + + +def test_route_to_vulkan_prebuilt_non_intel_unchanged(): + host = _host(is_linux = True, is_x86_64 = True) + routed, repo, _tag = ilp._route_to_vulkan_prebuilt(host, FORK, "", force_cpu = False) + assert repo == FORK + assert routed is host + + +def test_resolve_prebuilt_intel_host_routes_to_upstream(monkeypatch, capsys): + # The --resolve-prebuilt probe must agree with the install path: an + # auto-detected Intel host resolves against upstream (Vulkan), not the fork. + monkeypatch.setattr( + ilp, "detect_host", lambda: _host(is_linux = True, is_x86_64 = True, has_intel_gpu = True) + ) + seen, out = _run_resolve_capture_host(monkeypatch, capsys) + assert seen["repo"] == UPSTREAM + assert out["repo"] == UPSTREAM diff --git a/studio/backend/tests/test_llama_cpp_update.py b/studio/backend/tests/test_llama_cpp_update.py index 5138e90471..f405ebcbd1 100644 --- a/studio/backend/tests/test_llama_cpp_update.py +++ b/studio/backend/tests/test_llama_cpp_update.py @@ -448,6 +448,48 @@ def test_start_update_happy_path(monkeypatch, tmp_path): assert popen_kwargs["env"]["UNSLOTH_PROGRESS_PERCENT_STEP"] == "5" +def test_start_update_preserves_vulkan_via_env(monkeypatch, tmp_path): + # A Vulkan install (marker asset carries 'vulkan') must re-assert + # UNSLOTH_FORCE_VULKAN on update, or detect_host on a GPU box re-routes to + # CUDA/ROCm and silently replaces the Vulkan build. + install_dir = tmp_path / "llama.cpp" + binary = _write_install( + install_dir, + "b9493", + repo = "ggml-org/llama.cpp", + asset = "llama-b9493-bin-ubuntu-vulkan-x64.tar.gz", + ) + monkeypatch.setattr(upd, "_find_binary", lambda: binary) + monkeypatch.setattr(upd, "_installer_script", lambda: tmp_path / "install_llama_prebuilt.py") + monkeypatch.setattr(freshness, "_fetch_latest_release_tag", lambda repo, timeout = 5.0: "b9518") + + def _on_start(cmd): + _write_install( + install_dir, + "b9518", + repo = "ggml-org/llama.cpp", + asset = "llama-b9518-bin-ubuntu-vulkan-x64.tar.gz", + ) + + popen_kwargs: dict = {} + _patch_installer_popen( + monkeypatch, + lines = ["installed\n"], + on_start = _on_start, + captured_kwargs = popen_kwargs, + ) + + assert upd.start_update()["started"] is True + deadline = time.time() + 10 + while time.time() < deadline: + job = upd.get_update_status()["job"] + if job["state"] in ("success", "error"): + break + time.sleep(0.05) + assert job["state"] == "success", job + assert popen_kwargs["env"]["UNSLOTH_FORCE_VULKAN"] == "1" + + def test_start_update_reports_full_release_tag(monkeypatch, tmp_path): install_dir = tmp_path / "llama.cpp" binary = _write_install(install_dir, "b9595") diff --git a/studio/backend/tests/test_llama_cpp_vulkan_probe.py b/studio/backend/tests/test_llama_cpp_vulkan_probe.py new file mode 100644 index 0000000000..92aaab4873 --- /dev/null +++ b/studio/backend/tests/test_llama_cpp_vulkan_probe.py @@ -0,0 +1,193 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Vulkan free-VRAM reader regression tests on a synthetic probe output. + +Covers the post-probe handling in +``LlamaCppBackend._get_gpu_free_memory_vulkan``: + + * integrated GPUs (probe reports is_igpu=1) leave a flat per-device host + margin matching llama.cpp's --fit-target, so context auto-sizing can't + over-commit shared RAM, and report total 0 (shared RAM is not a budget), + * discrete GPUs (is_igpu=0) keep their free untouched and pass their real + total through so the fit can reserve absolute headroom, + * an inherited ``GGML_VK_VISIBLE_DEVICES`` is passed through to ggml unchanged + (ggml applies it), not stripped or filtered in Python -- the probe reports + ggml's compact ordinal, which load_model pins with ``--device Vulkan``. + +The ggml Vulkan library is never loaded: subprocess.run is mocked to emit +the tab-separated lines the real ``_vulkan_probe.py`` would print. +""" + +from __future__ import annotations + +import subprocess +import sys +import types as _types +from pathlib import Path +from unittest import mock + +import pytest + +_BACKEND_DIR = str(Path(__file__).resolve().parent.parent) +if _BACKEND_DIR not in sys.path: + sys.path.insert(0, _BACKEND_DIR) + +import importlib as _importlib # noqa: E402 + + +def _maybe_stub(name: str, builder): + try: + _importlib.import_module(name) + except ImportError: + sys.modules[name] = builder() + + +def _build_loggers_stub(): + m = _types.ModuleType("loggers") + m.get_logger = lambda name: __import__("logging").getLogger(name) + return m + + +_maybe_stub("loggers", _build_loggers_stub) +_maybe_stub("structlog", lambda: _types.ModuleType("structlog")) + +from core.inference import llama_cpp as _llama_mod # noqa: E402 +from core.inference.llama_cpp import ( # noqa: E402 + LlamaCppBackend, + _llama_lib_dir, + _vulkan_lib_filename, +) + +MIB = 1024 * 1024 +GIB = 1024 * MIB + + +def _make_vulkan_install(tmp_path: Path) -> str: + """A binary whose sibling dir holds the Vulkan ggml lib, so the + reader's ``is_vulkan_backend`` sibling-file check passes.""" + bindir = tmp_path / "build" / "bin" + bindir.mkdir(parents = True) + binary = bindir / ("llama-server.exe" if sys.platform == "win32" else "llama-server") + binary.write_bytes(b"stub") + (bindir / _vulkan_lib_filename()).write_bytes(b"stub") + return str(binary) + + +def _mock_probe(rows: list[str], captured_env: dict | None = None): + """Patch subprocess.run so the _vulkan_probe.py call returns ``rows`` + (already tab-formatted), recording the env it was launched with.""" + real_run = subprocess.run + + def fake_run(cmd, *args, **kwargs): + if isinstance(cmd, list) and any("_vulkan_probe" in str(c) for c in cmd): + if captured_env is not None: + captured_env.clear() + captured_env.update(kwargs.get("env") or {}) + return subprocess.CompletedProcess( + args = cmd, returncode = 0, stdout = "\n".join(rows), stderr = "" + ) + return real_run(cmd, *args, **kwargs) + + return mock.patch("subprocess.run", side_effect = fake_run) + + +def _row( + idx: int, + free_bytes: int, + is_igpu: int, + total_bytes: int = 0, +) -> str: + return f"{idx}\t{free_bytes}\t{is_igpu}\t{total_bytes}" + + +def test_integrated_gpu_leaves_host_margin(tmp_path): + binary = _make_vulkan_install(tmp_path) + # iGPU with 30 GiB free; reserve a flat 1024 MiB (llama.cpp --fit-target). + # total stays 0: shared system RAM is not a VRAM budget for the fit. + rows = [_row(0, 30 * GIB, is_igpu = 1, total_bytes = 32 * GIB)] + with _mock_probe(rows): + gpus = LlamaCppBackend._get_gpu_free_memory_vulkan(binary) + assert gpus == [(0, 30 * 1024 - 1024, 0)], gpus + + +def test_discrete_gpu_free_is_untouched_and_total_passed_through(tmp_path): + binary = _make_vulkan_install(tmp_path) + # 6 GiB free on a partially occupied 24 GiB card: free is untouched and the + # real total flows through so the fit reserves absolute headroom (CUDA/ROCm + # parity) instead of the looser free*frac budget. + rows = [_row(0, 6 * GIB, is_igpu = 0, total_bytes = 24 * GIB)] + with _mock_probe(rows): + gpus = LlamaCppBackend._get_gpu_free_memory_vulkan(binary) + assert gpus == [(0, 6 * 1024, 24 * 1024)], gpus + + +def test_large_discrete_gpu_is_untouched(tmp_path): + binary = _make_vulkan_install(tmp_path) + # A 48 GiB discrete card stays untouched regardless of size; only the + # iGPU flag triggers the host margin, never a VRAM/RAM ratio. + rows = [_row(0, 47 * GIB, is_igpu = 0, total_bytes = 48 * GIB)] + with _mock_probe(rows): + gpus = LlamaCppBackend._get_gpu_free_memory_vulkan(binary) + assert gpus == [(0, 47 * 1024, 48 * 1024)], gpus + + +def test_inherited_visible_devices_mask_is_passed_through_to_probe(tmp_path, monkeypatch): + # The mask is NOT stripped or filtered in Python: ggml parses it in raw + # physical-device space while this probe reports the compact post-filter + # ordinal, so mixing spaces would be wrong. It is passed through unchanged + # so ggml applies it to the same device list the launch will enumerate. + binary = _make_vulkan_install(tmp_path) + monkeypatch.setenv("GGML_VK_VISIBLE_DEVICES", "1") + captured: dict = {} + rows = [_row(0, 23 * GIB, is_igpu = 0, total_bytes = 24 * GIB)] + with _mock_probe(rows, captured_env = captured): + LlamaCppBackend._get_gpu_free_memory_vulkan(binary) + assert captured.get("GGML_VK_VISIBLE_DEVICES") == "1", captured + + +def test_vulkan_pin_args_uses_device_names_not_env_mask(): + # Pin by compact device name via --device (the space the probe reports and + # the registry names), never by writing a compact ordinal into the raw + # GGML_VK_VISIBLE_DEVICES index space. + assert LlamaCppBackend._vulkan_pin_args([0]) == ["--device", "Vulkan0"] + assert LlamaCppBackend._vulkan_pin_args([1, 2]) == ["--device", "Vulkan1,Vulkan2"] + assert LlamaCppBackend._vulkan_pin_args(None) == [] + assert LlamaCppBackend._vulkan_pin_args([]) == [] + + +def test_vulkan_only_build_is_detected(tmp_path): + binary = _make_vulkan_install(tmp_path) + assert LlamaCppBackend._is_vulkan_backend(binary) is True + + +def test_multi_backend_build_is_not_vulkan_only(tmp_path): + # A custom build that ships CUDA (or HIP) alongside Vulkan must NOT be + # treated as Vulkan-only, or its CUDA GPU would be probed/pinned as a Vulkan + # device; defer to the CUDA/HIP path instead. + binary = _make_vulkan_install(tmp_path) + cuda = "ggml-cuda.dll" if sys.platform == "win32" else "libggml-cuda.so" + (_llama_lib_dir(binary) / cuda).write_bytes(b"stub") + assert LlamaCppBackend._is_vulkan_backend(binary) is False + + +@pytest.mark.skipif(sys.platform == "win32", reason = "shell wrapper fallback is POSIX") +def test_shell_wrapper_entrypoint_resolves_to_real_lib_dir(tmp_path): + # create_exec_entrypoint falls back to a #!/bin/sh wrapper at the install root + # when it cannot symlink; _find_llama_server_binary returns that root entrypoint, + # so _llama_lib_dir must follow the wrapper's exec target to build/bin -- else + # _is_vulkan_backend misses libggml-vulkan.so and the Vulkan probe/pin silently + # never engage on a valid Vulkan install. + import os + + binary = _make_vulkan_install(tmp_path) # tmp_path/build/bin/llama-server + vulkan lib + bindir = Path(binary).parent + wrapper = tmp_path / "llama-server" + wrapper.write_text('#!/bin/sh\nexec "$(dirname "$0")/build/bin/llama-server" "$@"\n') + os.chmod(wrapper, 0o755) + assert _llama_lib_dir(str(wrapper)) == bindir + assert LlamaCppBackend._is_vulkan_backend(str(wrapper)) is True + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__, "-v"])) diff --git a/studio/backend/utils/llama_cpp_update.py b/studio/backend/utils/llama_cpp_update.py index c16ae91467..1bcbfbf95a 100644 --- a/studio/backend/utils/llama_cpp_update.py +++ b/studio/backend/utils/llama_cpp_update.py @@ -514,6 +514,12 @@ def _run_update(install_dir: Path, repo: str, asset: Optional[str], script: Path logger.info("llama update: installing", cmd = " ".join(cmd)) # Stream progress lines into job["progress"]. env = dict(os.environ, UNSLOTH_PROGRESS_PERCENT_STEP = "5") + # Preserve a Vulkan install across updates: detect_host on a CUDA/ROCm + # box would otherwise re-route and silently replace the Vulkan build. + # Re-assert it via the same env flag setup uses (mirrors + # _rocm_install_args). + if asset and "vulkan" in asset.lower(): + env["UNSLOTH_FORCE_VULKAN"] = "1" proc = subprocess.Popen( cmd, stdout = subprocess.PIPE, diff --git a/studio/install_llama_prebuilt.py b/studio/install_llama_prebuilt.py index 6c75e6c394..856ba71478 100644 --- a/studio/install_llama_prebuilt.py +++ b/studio/install_llama_prebuilt.py @@ -10,6 +10,7 @@ import argparse import atexit import errno import fnmatch +import glob import hashlib import json import os @@ -265,6 +266,7 @@ class HostInfo: has_physical_nvidia: bool has_usable_nvidia: bool has_rocm: bool = False + has_intel_gpu: bool = False rocm_gfx_target: str | None = None # (major, minor) from platform.mac_ver(); None off macOS or if unparseable. # Skips a macos prebuilt whose minimum-OS exceeds this host. @@ -1482,6 +1484,24 @@ def direct_upstream_release_plan( install_kind = "windows-hip", ) ) + # Intel (or other non-NVIDIA/non-AMD) GPU: use the Vulkan prebuilt. Gate + # on no PHYSICAL NVIDIA (not just no usable one): a host that hid NVIDIA + # via CUDA_VISIBLE_DEVICES must not reach Vulkan, which ignores that mask + # and could enumerate the reserved card. Falls through to CPU below. + elif host.has_intel_gpu and not host.has_physical_nvidia: + vulkan_asset = f"llama-{release_tag}-bin-win-vulkan-x64.zip" + vulkan_url = assets.get(vulkan_asset) + if vulkan_url: + attempts.append( + AssetChoice( + repo = repo, + tag = release_tag, + name = vulkan_asset, + url = vulkan_url, + source_label = "upstream", + install_kind = "windows-vulkan", + ) + ) cpu_asset = f"llama-{release_tag}-bin-win-cpu-x64.zip" cpu_url = assets.get(cpu_asset) if cpu_url: @@ -1545,6 +1565,23 @@ def direct_upstream_release_plan( # ROCm hosts are excluded: this ggml-org path ships no per-gfx ROCm # asset, so they fall through to the empty-attempts raise (HIP source # build) rather than silently getting a CPU binary on a GPU host. + # Intel (or other non-NVIDIA/non-AMD) GPU: use the Vulkan prebuilt. The + # elif already excludes usable NVIDIA and ROCm; also require no PHYSICAL + # NVIDIA so a CUDA-hidden card isn't reached through Vulkan (CPU below). + if host.has_intel_gpu and not host.has_physical_nvidia: + vulkan_asset = f"llama-{release_tag}-bin-ubuntu-vulkan-x64.tar.gz" + vulkan_url = assets.get(vulkan_asset) + if vulkan_url: + attempts.append( + AssetChoice( + repo = repo, + tag = release_tag, + name = vulkan_asset, + url = vulkan_url, + source_label = "upstream", + install_kind = "linux-vulkan", + ) + ) asset_name = f"llama-{release_tag}-bin-ubuntu-x64.tar.gz" asset_url = assets.get(asset_name) if asset_url: @@ -1564,6 +1601,23 @@ def direct_upstream_release_plan( # selector returned 0 attempts and the installer fell back to a # source build on every Linux ARM64 host (DGX Spark, Ampere # Altra, GitHub-hosted ubuntu-24.04-arm runners, etc.). + # Intel (or other non-NVIDIA/non-AMD) GPU: prefer the Vulkan prebuilt, + # mirroring the x86_64 branch. Upstream ships bin-ubuntu-vulkan-arm64. + # No physical NVIDIA: don't reach a CUDA-hidden card through Vulkan. + if host.has_intel_gpu and not host.has_physical_nvidia and not host.has_rocm: + vulkan_asset = f"llama-{release_tag}-bin-ubuntu-vulkan-arm64.tar.gz" + vulkan_url = assets.get(vulkan_asset) + if vulkan_url: + attempts.append( + AssetChoice( + repo = repo, + tag = release_tag, + name = vulkan_asset, + url = vulkan_url, + source_label = "upstream", + install_kind = "linux-vulkan", + ) + ) asset_name = f"llama-{release_tag}-bin-ubuntu-arm64.tar.gz" asset_url = assets.get(asset_name) if asset_url: @@ -3075,6 +3129,40 @@ def detect_host() -> HostInfo: # Note: amdhip64.dll presence alone is NOT treated as GPU evidence # since the HIP SDK can be installed without an AMD GPU. + # Detect an Intel GPU; gates the Vulkan prebuilt. Linux reads the DRM sysfs + # vendor id (0x8086); Windows queries the WMI video controller list. Only + # probed with no usable NVIDIA and no ROCm (matching the Vulkan branches), + # keeping the probe (notably the Windows powershell call) off that path. + has_intel_gpu = False + if not has_usable_nvidia and not has_rocm: + if is_linux: + for _vendor_file in glob.glob("/sys/class/drm/card*/device/vendor"): + try: + with open(_vendor_file) as _vf: + if _vf.read().strip().lower() == "0x8086": + has_intel_gpu = True + break + except OSError: + continue + elif is_windows: + _ps = shutil.which("powershell") or shutil.which("pwsh") + if _ps: + try: + _result = run_capture( + [ + _ps, + "-NoProfile", + "-Command", + "Get-CimInstance Win32_VideoController | " + "Select-Object -ExpandProperty Name", + ], + timeout = 15, + ) + if _result.returncode == 0 and "intel" in _result.stdout.lower(): + has_intel_gpu = True + except Exception: + pass + return HostInfo( system = system, machine = machine, @@ -3090,6 +3178,7 @@ def detect_host() -> HostInfo: has_physical_nvidia = has_physical_nvidia, has_usable_nvidia = has_usable_nvidia, has_rocm = has_rocm, + has_intel_gpu = has_intel_gpu, rocm_gfx_target = rocm_gfx_target, macos_version = macos_version, ) @@ -3126,6 +3215,7 @@ def _apply_host_overrides( has_physical_nvidia = False, has_rocm = False, rocm_gfx_target = None, + has_intel_gpu = False, ) gfx = _normalize_forwarded_gfx(override_rocm_gfx) if gfx: @@ -3866,6 +3956,23 @@ def resolve_upstream_asset_choice(host: HostInfo, llama_tag: str) -> AssetChoice "falling back to source build with HIP support" ) + # Intel (or other non-NVIDIA/non-AMD) GPU: use the Vulkan prebuilt. No + # physical NVIDIA (not just no usable one): a CUDA-hidden card must not + # be reached through Vulkan, which ignores CUDA_VISIBLE_DEVICES. + if host.has_intel_gpu and not host.has_physical_nvidia and not host.has_rocm: + vulkan_name = f"llama-{llama_tag}-bin-ubuntu-vulkan-x64.tar.gz" + if vulkan_name in upstream_assets: + log(f"Intel GPU detected -- using upstream Vulkan prebuilt {vulkan_name}") + return AssetChoice( + repo = UPSTREAM_REPO, + tag = llama_tag, + name = vulkan_name, + url = upstream_assets[vulkan_name], + source_label = "upstream", + install_kind = "linux-vulkan", + ) + log("Intel GPU detected but no Vulkan prebuilt found -- falling back to CPU") + upstream_name = f"llama-{llama_tag}-bin-ubuntu-x64.tar.gz" if upstream_name not in upstream_assets: raise PrebuiltFallback("upstream Linux CPU asset was not found") @@ -3908,6 +4015,24 @@ def resolve_upstream_asset_choice(host: HostInfo, llama_tag: str) -> AssetChoice ) log("AMD ROCm detected on Windows but no HIP prebuilt found -- falling back to CPU") + # Intel (or other non-NVIDIA/non-AMD) GPU on Windows: use Vulkan. No + # physical NVIDIA so a CUDA-hidden card isn't reached through Vulkan. + if host.has_intel_gpu and not host.has_physical_nvidia and not host.has_rocm: + vulkan_name = f"llama-{llama_tag}-bin-win-vulkan-x64.zip" + if vulkan_name in upstream_assets: + log( + f"Intel GPU detected on Windows -- using upstream Vulkan prebuilt {vulkan_name}" + ) + return AssetChoice( + repo = UPSTREAM_REPO, + tag = llama_tag, + name = vulkan_name, + url = upstream_assets[vulkan_name], + source_label = "upstream", + install_kind = "windows-vulkan", + ) + log("Intel GPU detected on Windows but no Vulkan prebuilt found -- falling back to CPU") + upstream_name = f"llama-{llama_tag}-bin-win-cpu-x64.zip" if upstream_name not in upstream_assets: raise PrebuiltFallback("upstream Windows CPU asset was not found") @@ -4503,6 +4628,7 @@ def runtime_patterns_for_choice(choice: AssetChoice) -> list[str]: "linux-arm64-cuda", "linux-rocm", "linux-arm64", + "linux-vulkan", }: return ["llama-server", "llama-quantize", "llama-diffusion-gemma-visual-server", "lib*.so*"] if choice.install_kind in {"macos-arm64", "macos-x64"}: @@ -4516,6 +4642,7 @@ def runtime_patterns_for_choice(choice: AssetChoice) -> list[str]: "windows-cpu", "windows-cuda", "windows-hip", + "windows-vulkan", "windows-rocm", "windows-arm64", }: @@ -5731,8 +5858,10 @@ def validate_server( "linux-cuda", "linux-arm64-cuda", "linux-rocm", + "linux-vulkan", "windows-cuda", "windows-hip", + "windows-vulkan", "windows-rocm", "macos-arm64", } @@ -6354,6 +6483,20 @@ def runtime_payload_health_groups(choice: AssetChoice) -> list[list[str]]: ["libmtmd.so*"], ["libggml-hip.so*"], ] + if choice.install_kind == "linux-vulkan": + return [ + ["libllama-common.so*"], + ["libllama.so*"], + ["libggml.so*"], + ["libggml-base.so*"], + # Match the sibling globs (linux-cuda/-rocm): x64 bundles ship + # arch-suffixed libggml-cpu-.so, arm64 may ship a bare + # libggml-cpu.so; the '-' form missed the latter and re-flagged + # the install unhealthy on every check. + ["libggml-cpu*.so*"], + ["libmtmd.so*"], + ["libggml-vulkan.so*"], + ] if choice.install_kind in {"windows-cpu", "windows-arm64"}: return [["llama.dll"]] if choice.install_kind == "windows-cuda": @@ -6373,6 +6516,8 @@ def runtime_payload_health_groups(choice: AssetChoice) -> list[list[str]]: return groups if choice.install_kind in {"windows-hip", "windows-rocm"}: return [["llama.dll"], ["*hip*.dll"]] + if choice.install_kind == "windows-vulkan": + return [["llama.dll"], ["ggml-vulkan.dll"]] return [] @@ -6654,6 +6799,89 @@ def validate_prebuilt_attempts( raise PrebuiltFallback("no prebuilt bundle passed validation") +def force_vulkan_requested() -> bool: + """Whether UNSLOTH_FORCE_VULKAN opts this host into the Vulkan llama.cpp + prebuilt instead of its detected CUDA/ROCm backend (e.g. so an AMD user can + run the Vulkan build for inference). Scoped to the llama.cpp backend; the + torch/training stack installs separately and still sees the real GPU. + """ + return os.environ.get("UNSLOTH_FORCE_VULKAN", "").strip().lower() in ( + "1", + "true", + "yes", + ) + + +def _vulkan_only_host(host: HostInfo) -> HostInfo: + """Rewrite ``host`` so the asset selectors take their Vulkan branch. + + That branch fires on ``has_intel_gpu and not nvidia and not rocm``, so clear + the CUDA/ROCm flags and raise the integrated-GPU flag. The synthetic flag + never leaves install planning -- it only routes the llama.cpp prebuilt + choice, not the torch/training stack. + """ + return dataclasses_replace( + host, + has_usable_nvidia = False, + has_physical_nvidia = False, + has_rocm = False, + has_intel_gpu = True, + ) + + +def _route_to_vulkan_prebuilt( + host: HostInfo, published_repo: str, published_release_tag: str, *, force_cpu: bool +) -> tuple[HostInfo, str, str]: + """Point a Vulkan-capable host at the upstream ggml-org Vulkan prebuilt. + + The unsloth published repo ships only CUDA/ROCm/CPU assets, so Vulkan comes + from UPSTREAM_REPO. Two triggers route here, both suppressed under + --cpu-fallback (the explicit "give me CPU" last resort wins): + * UNSLOTH_FORCE_VULKAN forces Vulkan over the detected CUDA/ROCm backend; + * an auto-detected Intel GPU with NO physical NVIDIA/ROCm -- the purpose + of the has_intel_gpu probe, since the fork manifest ships no Vulkan asset. + Applied by BOTH the install path and the --resolve-prebuilt probe so the + "is a prebuilt available" answer matches what actually gets installed. + + Returns the (possibly rewritten) host, repo, and release tag. + """ + forced = force_vulkan_requested() + # Gate auto-routing on no PHYSICAL NVIDIA, not merely no usable one: a mixed + # NVIDIA+Intel host that hides NVIDIA with CUDA_VISIBLE_DEVICES=""/-1 keeps + # has_physical_nvidia=True while has_usable_nvidia goes False. Vulkan ignores + # CUDA_VISIBLE_DEVICES, so auto-routing such a host would let it grab the + # reserved NVIDIA GPU. An explicit UNSLOTH_FORCE_VULKAN still overrides. + auto_intel = host.has_intel_gpu and not host.has_physical_nvidia and not host.has_rocm + if force_cpu or not (forced or auto_intel): + return host, published_repo, published_release_tag + if host.is_macos: + if forced: + log( + "UNSLOTH_FORCE_VULKAN is set but ignored on macOS " + "(Metal is used; there is no Vulkan prebuilt)" + ) + return host, published_repo, published_release_tag + if forced: + log( + "UNSLOTH_FORCE_VULKAN is set; installing the upstream Vulkan " + "llama.cpp prebuilt instead of the detected GPU backend" + ) + # Forcing may override a detected NVIDIA/ROCm host, so normalize it to + # Vulkan-only; an auto-detected Intel host already is. + host = _vulkan_only_host(host) + else: + log("Intel GPU detected; installing the upstream Vulkan llama.cpp prebuilt") + # Swapping the fork for upstream invalidates a fork release pin: the two use + # different tag namespaces (fork b9596-mix- vs upstream b9596), so a + # pinned fork tag would make the upstream resolver query a nonexistent + # release and fall back to source. Drop it and let the upstream resolver + # pick by the requested llama tag. A pin already on an explicit upstream repo + # (repo unchanged here) is preserved. + if published_repo != UPSTREAM_REPO: + published_release_tag = "" + return host, UPSTREAM_REPO, published_release_tag + + def diffusion_visual_server_backfill_needed( install_dir: Path, host: HostInfo, choice: AssetChoice ) -> bool: @@ -6696,6 +6924,9 @@ def install_prebuilt( override_rocm_gfx = override_rocm_gfx, force_cpu = force_cpu, ) + host, published_repo, published_release_tag = _route_to_vulkan_prebuilt( + host, published_repo, published_release_tag, force_cpu = force_cpu + ) choice: AssetChoice | None = None try: with install_lock(install_lock_path(install_dir)): @@ -6708,7 +6939,9 @@ def install_prebuilt( f"no existing llama.cpp install detected at {install_dir}; performing fresh prebuilt install" ) # Single resolver: every fork host selects from the release manifest; - # an explicit ggml-org override selects by asset filename instead. + # an explicit ggml-org override selects by asset filename instead. A + # forced-Vulkan host already has published_repo pointed at + # UPSTREAM_REPO above, so the resolver takes the Vulkan asset branch. requested_tag, release_plans = resolve_simple_install_release_plans( llama_tag, host, @@ -6994,10 +7227,14 @@ def main() -> int: override_rocm_gfx = args.rocm_gfx, force_cpu = args.cpu_fallback, ) - repo = args.published_repo + # Same Vulkan routing the install path applies, so the probe's answer + # matches what would install (an Intel/forced-Vulkan host -> upstream). + host, repo, release_tag = _route_to_vulkan_prebuilt( + host, args.published_repo, args.published_release_tag or "", force_cpu = args.cpu_fallback + ) try: _requested, plans = resolve_simple_install_release_plans( - args.resolve_prebuilt, host, repo, args.published_release_tag or "" + args.resolve_prebuilt, host, repo, release_tag ) choice = plans[0].attempts[0] if plans and plans[0].attempts else None if choice is None: From 216a1fad33561ee4fcf24fd47811fbf721b46f29 Mon Sep 17 00:00:00 2001 From: alkinun Date: Thu, 9 Jul 2026 13:46:47 +0300 Subject: [PATCH 018/372] Fix Windows installer torch index override (#6972) * Fix Windows installer torch index override * Clear inherited uv index env vars for pinned installs in studio/setup.ps1 (#6898) * Harden setup.ps1 index-var clearing to truly remove vars (#6898) * Apply UV_DEFAULT_INDEX torch index fix to Linux/Mac install.sh (#6898) * Neutralize all uv index env vars for pinned torch installs (#6898) * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: Daniel Han Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- install.ps1 | 24 ++++++--- install.sh | 28 ++++++---- studio/setup.ps1 | 13 ++++- .../test_tokenizers_and_torch_constraint.py | 51 +++++++++++++++++++ tests/sh/test_mac_intel_compat.sh | 2 +- 5 files changed, 99 insertions(+), 19 deletions(-) diff --git a/install.ps1 b/install.ps1 index 696f4e613a..0797cd3868 100644 --- a/install.ps1 +++ b/install.ps1 @@ -469,6 +469,17 @@ function Install-UnslothStudio { param( [Parameter(Mandatory = $true)][ScriptBlock]$Command ) + # Installer-pinned index installs (torch) must beat an inherited uv mirror + # (#6898): when the command pins an index, clear every uv index env var so + # it wins, then restore in finally. Other installs keep the user's mirror. + $savedUvIndex = $null + if ($Command.ToString() -match '--default-index') { + $savedUvIndex = @{} + foreach ($n in 'UV_DEFAULT_INDEX', 'UV_INDEX_URL', 'UV_INDEX', 'UV_EXTRA_INDEX_URL') { + $savedUvIndex[$n] = [Environment]::GetEnvironmentVariable($n) + Remove-Item "Env:$n" -ErrorAction SilentlyContinue + } + } $prevEap = $ErrorActionPreference $ErrorActionPreference = "Continue" try { @@ -488,6 +499,7 @@ function Install-UnslothStudio { return [int]$LASTEXITCODE } finally { $ErrorActionPreference = $prevEap + if ($savedUvIndex) { foreach ($n in $savedUvIndex.Keys) { if ($null -ne $savedUvIndex[$n]) { Set-Item "Env:$n" $savedUvIndex[$n] } } } } } @@ -2200,7 +2212,7 @@ exit 0 # ABI-incompatible torchvision/torchaudio on AMD's per-arch index. $visionSpec = if ($ROCmGfxArch -and $torchvisionFloorMap.ContainsKey($ROCmGfxArch)) { $torchvisionFloorMap[$ROCmGfxArch] } else { "torchvision" } $audioSpec = if ($ROCmGfxArch -and $torchaudioFloorMap.ContainsKey($ROCmGfxArch)) { $torchaudioFloorMap[$ROCmGfxArch] } else { "torchaudio" } - $torchInstallExit = Invoke-InstallCommandRetry -Label "install PyTorch (AMD ROCm)" { uv pip install --python $VenvPython --force-reinstall --index-url $ROCmIndexUrl $torchSpec $visionSpec $audioSpec } + $torchInstallExit = Invoke-InstallCommandRetry -Label "install PyTorch (AMD ROCm)" { uv pip install --python $VenvPython --force-reinstall --default-index $ROCmIndexUrl $torchSpec $visionSpec $audioSpec } if ($torchInstallExit -ne 0) { # Transient AMD-index failure: fall back to a CPU base so the install # still completes; Studio setup retries ROCm afterwards. @@ -2209,7 +2221,7 @@ exit 0 # torch (e.g. 2.10.0+rocm on gfx110X/gfx90a) that still satisfies the CPU # torch>= range, so without it uv would keep the ROCm build and only swap # the companions -- a mismatched venv the flavor-repair block won't fix. - $torchInstallExit = Invoke-InstallCommandRetry -Label "install PyTorch (CPU fallback)" { uv pip install --python $VenvPython --force-reinstall "torch>=2.4,<2.11.0" torchvision torchaudio --index-url $TorchIndexUrl } + $torchInstallExit = Invoke-InstallCommandRetry -Label "install PyTorch (CPU fallback)" { uv pip install --python $VenvPython --force-reinstall "torch>=2.4,<2.11.0" torchvision torchaudio --default-index $TorchIndexUrl } if ($torchInstallExit -ne 0) { Write-Host "[ERROR] Failed to install PyTorch (ROCm and CPU base both failed, exit code $torchInstallExit)" -ForegroundColor Red return (Exit-InstallFailure "Failed to install PyTorch (exit code $torchInstallExit)" $torchInstallExit) @@ -2223,7 +2235,7 @@ exit 0 } else { Write-TauriLog "STEP" "Installing PyTorch" substep "installing PyTorch ($TorchIndexUrl)..." - $torchInstallExit = Invoke-InstallCommandRetry -Label "install PyTorch" { uv pip install --python $VenvPython "torch>=2.4,<2.11.0" torchvision torchaudio --index-url $TorchIndexUrl } + $torchInstallExit = Invoke-InstallCommandRetry -Label "install PyTorch" { uv pip install --python $VenvPython "torch>=2.4,<2.11.0" torchvision torchaudio --default-index $TorchIndexUrl } if ($torchInstallExit -ne 0) { Write-Host "[ERROR] Failed to install PyTorch (exit code $torchInstallExit)" -ForegroundColor Red return (Exit-InstallFailure "Failed to install PyTorch (exit code $torchInstallExit)" $torchInstallExit) @@ -2306,7 +2318,7 @@ exit 0 # keeps a stale torch==X+cpu against a CUDA index and setup.ps1 then loops on # "torch cpu != required cuXXX". Reinstall the right triplet when a GPU build is # expected: CUDA from $TorchIndexUrl, ROCm from $ROCmIndexUrl (repo.amd.com gfx* - # is a PEP 503 index uv resolves via --index-url, same URL the fresh ROCm install + # is a PEP 503 index uv resolves via --default-index, same URL the fresh ROCm install # above uses). --no-torch / CPU-only hosts (expected cpu) are no-ops. if (-not $SkipTorch) { $expectedTorchTag = Get-ExpectedTorchFlavorTag -TorchIndexUrl $TorchIndexUrl -ROCmIndexUrl $ROCmIndexUrl @@ -2322,7 +2334,7 @@ exit 0 $visionSpec = if ($ROCmGfxArch -and $torchvisionFloorMap.ContainsKey($ROCmGfxArch)) { $torchvisionFloorMap[$ROCmGfxArch] } else { "torchvision" } $audioSpec = if ($ROCmGfxArch -and $torchaudioFloorMap.ContainsKey($ROCmGfxArch)) { $torchaudioFloorMap[$ROCmGfxArch] } else { "torchaudio" } substep "PyTorch flavor mismatch (installed $installedTorchTag, need ROCm) -- reinstalling correct build..." "Yellow" - $torchFixExit = Invoke-InstallCommand { uv pip install --python $VenvPython --force-reinstall --index-url $ROCmIndexUrl $rocmSpec $visionSpec $audioSpec } + $torchFixExit = Invoke-InstallCommand { uv pip install --python $VenvPython --force-reinstall --default-index $ROCmIndexUrl $rocmSpec $visionSpec $audioSpec } if ($torchFixExit -ne 0) { Write-Host "[ERROR] Failed to reinstall PyTorch with the correct ROCm build (exit code $torchFixExit)" -ForegroundColor Red return (Exit-InstallFailure "Failed to reinstall PyTorch (ROCm) (exit code $torchFixExit)" $torchFixExit) @@ -2331,7 +2343,7 @@ exit 0 } elseif ($expectedTorchTag -ne 'rocm') { # CUDA: stale +cpu (or wrong cuXXX) against a CUDA index -> reinstall triplet. substep "PyTorch flavor mismatch (installed $installedTorchTag, need $expectedTorchTag) -- reinstalling correct build..." "Yellow" - $torchFixExit = Invoke-InstallCommand { uv pip install --python $VenvPython "torch>=2.4,<2.11.0" torchvision torchaudio --index-url $TorchIndexUrl --reinstall-package torch --reinstall-package torchvision --reinstall-package torchaudio } + $torchFixExit = Invoke-InstallCommand { uv pip install --python $VenvPython "torch>=2.4,<2.11.0" torchvision torchaudio --default-index $TorchIndexUrl --reinstall-package torch --reinstall-package torchvision --reinstall-package torchaudio } if ($torchFixExit -ne 0) { Write-Host "[ERROR] Failed to reinstall PyTorch with the correct CUDA build (exit code $torchFixExit)" -ForegroundColor Red return (Exit-InstallFailure "Failed to reinstall PyTorch ($expectedTorchTag) (exit code $torchFixExit)" $torchFixExit) diff --git a/install.sh b/install.sh index 0acc9ec0be..3f4ea92387 100755 --- a/install.sh +++ b/install.sh @@ -159,6 +159,12 @@ run_maybe_quiet() { run_install_cmd() { _label="$1" shift + # Installer-pinned index installs (torch) must beat an inherited uv mirror + # (#6898): when we pass --default-index, neutralize every uv index env var so + # the pinned index wins. Other installs keep the user's mirror. + case " $* " in + *" --default-index "*) set -- env -u UV_DEFAULT_INDEX -u UV_INDEX_URL -u UV_INDEX -u UV_EXTRA_INDEX_URL "$@" ;; + esac if _is_verbose; then "$@" && return 0 _rc=$? @@ -2190,9 +2196,9 @@ _expected_torch_flavor_tag() { esac } -# Whether index ($1) supports a plain --index-url reinstall. pytorch.org cuXXX / +# Whether index ($1) supports a plain --default-index reinstall. pytorch.org cuXXX / # rocmX.Y AND the repo.amd.com gfx* indexes are all PEP 503 simple indexes that uv -# resolves (torch + every transitive dep) via --index-url -- the same URLs the +# resolves (torch + every transitive dep) via --default-index -- the same URLs the # fresh-install paths above already use -- so a stale wheel is auto-repairable. # Unknown/odd-mirror leaves -> no, so we warn rather than risk a wrong reinstall. _torch_index_repairable() { @@ -2744,7 +2750,7 @@ if [ "$_MIGRATED" = true ]; then substep "repairing ROCm torch (overwritten by dependency resolution)..." run_install_cmd_retry "repair ROCm torch" uv pip install --python "$_VENV_PY" \ "$TORCH_CONSTRAINT" torchvision torchaudio \ - --index-url "$TORCH_INDEX_URL" \ + --default-index "$TORCH_INDEX_URL" \ --force-reinstall fi ;; @@ -2870,7 +2876,7 @@ elif [ -n "$TORCH_INDEX_URL" ]; then substep "[WARN] Radeon repo lacks a compatible wheel set for this Python; falling back to ROCm index ($TORCH_INDEX_URL)" "$C_WARN" run_install_cmd_retry "install PyTorch" uv pip install --python "$_VENV_PY" \ "$TORCH_CONSTRAINT" torchvision torchaudio \ - --index-url "$TORCH_INDEX_URL" + --default-index "$TORCH_INDEX_URL" else substep "installing PyTorch from Radeon repo (${_RADEON_BASE_URL})..." # Pass explicit wheel URLs so the matched trio is @@ -2893,18 +2899,18 @@ elif [ -n "$TORCH_INDEX_URL" ]; then substep "[WARN] Radeon repo unavailable; falling back to ROCm index ($TORCH_INDEX_URL)" "$C_WARN" run_install_cmd_retry "install PyTorch" uv pip install --python "$_VENV_PY" \ "$TORCH_CONSTRAINT" torchvision torchaudio \ - --index-url "$TORCH_INDEX_URL" + --default-index "$TORCH_INDEX_URL" fi else substep "[WARN] Radeon GPU detected but could not detect full ROCm version; falling back to ROCm index" "$C_WARN" run_install_cmd_retry "install PyTorch" uv pip install --python "$_VENV_PY" \ "$TORCH_CONSTRAINT" torchvision torchaudio \ - --index-url "$TORCH_INDEX_URL" + --default-index "$TORCH_INDEX_URL" fi else substep "installing PyTorch ($TORCH_INDEX_URL)..." run_install_cmd_retry "install PyTorch" uv pip install --python "$_VENV_PY" "$TORCH_CONSTRAINT" torchvision torchaudio \ - --index-url "$TORCH_INDEX_URL" + --default-index "$TORCH_INDEX_URL" fi # AMD ROCm: install bitsandbytes (once, after torch, for all ROCm paths). # Gate on SKIP_TORCH=false so a user running with --no-torch on a ROCm @@ -2964,7 +2970,7 @@ elif [ -n "$TORCH_INDEX_URL" ]; then substep "repairing ROCm torch (overwritten by dependency resolution)..." run_install_cmd_retry "repair ROCm torch" uv pip install --python "$_VENV_PY" \ "$TORCH_CONSTRAINT" torchvision torchaudio \ - --index-url "$TORCH_INDEX_URL" \ + --default-index "$TORCH_INDEX_URL" \ --force-reinstall fi ;; @@ -2999,14 +3005,14 @@ if [ "$SKIP_TORCH" = false ] && [ -n "${TORCH_INDEX_URL:-}" ]; then _installed_torch_ver=$("$_VENV_PY" -c "import torch; print(torch.__version__)" 2>/dev/null || true) _installed_torch_tag="" [ -n "$_installed_torch_ver" ] && _installed_torch_tag=$(_torch_flavor_tag "$_installed_torch_ver") - # Repair when flavor is wrong AND the index is plain --index-url reinstallable + # Repair when flavor is wrong AND the index is plain --default-index reinstallable # (cuXXX / rocmX.Y / repo.amd.com gfx*); an unknown mirror leaf -> warn only. if [ -n "$_installed_torch_tag" ] && [ "$_installed_torch_tag" != "$_expected_torch_tag" ] \ && [ "$(_torch_index_repairable "$TORCH_INDEX_URL")" = "yes" ]; then substep "PyTorch flavor mismatch (installed $_installed_torch_tag, need $_expected_torch_tag) -- reinstalling correct build..." run_install_cmd "reinstall PyTorch ($_expected_torch_tag)" uv pip install --python "$_VENV_PY" \ "$TORCH_CONSTRAINT" torchvision torchaudio \ - --index-url "$TORCH_INDEX_URL" \ + --default-index "$TORCH_INDEX_URL" \ --reinstall-package torch --reinstall-package torchvision --reinstall-package torchaudio _installed_torch_ver=$("$_VENV_PY" -c "import torch; print(torch.__version__)" 2>/dev/null || true) _installed_torch_tag="" @@ -3017,7 +3023,7 @@ if [ "$SKIP_TORCH" = false ] && [ -n "${TORCH_INDEX_URL:-}" ]; then substep "[WARN] PyTorch is CPU-only but a $_expected_torch_tag GPU build was expected for this machine." "$C_WARN" substep "[WARN] Training and GPU inference will run on CPU until this is fixed." "$C_WARN" substep "[WARN] Re-run this installer, or reinstall the GPU build manually:" "$C_WARN" - substep "[WARN] uv pip install --python \"$_VENV_PY\" \"$TORCH_CONSTRAINT\" torchvision torchaudio --index-url $TORCH_INDEX_URL --reinstall-package torch --reinstall-package torchvision --reinstall-package torchaudio" "$C_WARN" + substep "[WARN] uv pip install --python \"$_VENV_PY\" \"$TORCH_CONSTRAINT\" torchvision torchaudio --default-index $TORCH_INDEX_URL --reinstall-package torch --reinstall-package torchvision --reinstall-package torchaudio" "$C_WARN" fi fi fi diff --git a/studio/setup.ps1 b/studio/setup.ps1 index 07dcb17335..db01a1ecad 100644 --- a/studio/setup.ps1 +++ b/studio/setup.ps1 @@ -2621,7 +2621,18 @@ function Fast-Install { param([Parameter(ValueFromRemainingArguments=$true)]$Args_) if ($UseUv) { $VenvPy = (Get-Command python).Source - $result = & uv pip install --python $VenvPy @Args_ 2>&1 + # An explicit --index-url must win. Inherited uv index env vars otherwise + # override it and pull CPU torch over the CUDA/ROCm build (#6898), so drop + # them only for index-pinned installs; mirrors still apply elsewhere. + $saved = @{} + if (@($Args_) -contains '--index-url') { + foreach ($n in 'UV_DEFAULT_INDEX', 'UV_INDEX_URL', 'UV_INDEX', 'UV_EXTRA_INDEX_URL') { + $saved[$n] = [Environment]::GetEnvironmentVariable($n) + Remove-Item "Env:$n" -ErrorAction SilentlyContinue + } + } + try { $result = & uv pip install --python $VenvPy @Args_ 2>&1 } + finally { foreach ($n in $saved.Keys) { if ($null -ne $saved[$n]) { Set-Item "Env:$n" $saved[$n] } } } if ($LASTEXITCODE -eq 0) { return } } & python -m pip install @Args_ 2>&1 diff --git a/tests/python/test_tokenizers_and_torch_constraint.py b/tests/python/test_tokenizers_and_torch_constraint.py index 7390d7be9b..4322f0c7d6 100644 --- a/tests/python/test_tokenizers_and_torch_constraint.py +++ b/tests/python/test_tokenizers_and_torch_constraint.py @@ -14,6 +14,7 @@ _TESTS_DIR = pathlib.Path(__file__).resolve().parent.parent # tests/ _REPO_ROOT = _TESTS_DIR.parent # unsloth/ _INSTALL_SH = _REPO_ROOT / "install.sh" _INSTALL_PS1 = _REPO_ROOT / "install.ps1" +_SETUP_PS1 = _REPO_ROOT / "studio" / "setup.ps1" _NO_TORCH_RT = _REPO_ROOT / "studio" / "backend" / "requirements" / "no-torch-runtime.txt" @@ -109,6 +110,56 @@ class TestStructuralInstallPs1Unchanged: assert '"torch>=2.4,<2.11.0"' in self._ps1 +class TestInstallPs1UvDefaultIndex: + """Installer-managed torch indexes must override inherited uv defaults.""" + + _ps1 = _read(_INSTALL_PS1) + + def test_torch_installs_use_default_index(self): + assert "--default-index $TorchIndexUrl" in self._ps1 + assert "--default-index $ROCmIndexUrl" in self._ps1 + + def test_torch_installs_do_not_use_deprecated_index_url(self): + assert "--index-url $TorchIndexUrl" not in self._ps1 + assert "--index-url $ROCmIndexUrl" not in self._ps1 + + def test_torch_installs_neutralize_all_uv_index_env_vars(self): + # Extra-index vars outrank --default-index, so pinned installs must clear them. + for var in ("UV_DEFAULT_INDEX", "UV_INDEX_URL", "UV_INDEX", "UV_EXTRA_INDEX_URL"): + assert var in self._ps1 + assert 'Remove-Item "Env:$n"' in self._ps1 + + +class TestSetupPs1FastInstallIndex: + """setup.ps1 Fast-Install must neutralize inherited uv indexes when pinning.""" + + _ps1 = _read(_SETUP_PS1) + + def test_fast_install_clears_all_uv_index_env_vars(self): + for var in ("UV_DEFAULT_INDEX", "UV_INDEX_URL", "UV_INDEX", "UV_EXTRA_INDEX_URL"): + assert var in self._ps1 + # Must truly remove the vars (child sees no value), not set them empty. + assert 'Remove-Item "Env:$n"' in self._ps1 + + +class TestInstallShUvDefaultIndex: + """Linux/Mac installer torch indexes must override inherited uv defaults.""" + + _sh = _read(_INSTALL_SH) + + def test_torch_installs_use_default_index(self): + assert '--default-index "$TORCH_INDEX_URL"' in self._sh + + def test_torch_installs_do_not_use_deprecated_index_url(self): + assert '--index-url "$TORCH_INDEX_URL"' not in self._sh + + def test_torch_installs_neutralize_all_uv_index_env_vars(self): + # --default-index installs run with all uv index env vars unset via `env -u`. + assert ( + "env -u UV_DEFAULT_INDEX -u UV_INDEX_URL -u UV_INDEX -u UV_EXTRA_INDEX_URL" in self._sh + ) + + # Group 2 -- Shell snippet tests (bash subprocess, mocked python) class TestTorchConstraintShell: """Test the TORCH_CONSTRAINT block via bash with mocked python minor versions.""" diff --git a/tests/sh/test_mac_intel_compat.sh b/tests/sh/test_mac_intel_compat.sh index 3c3bbfaa5f..8a0ff4b641 100644 --- a/tests/sh/test_mac_intel_compat.sh +++ b/tests/sh/test_mac_intel_compat.sh @@ -312,7 +312,7 @@ if [ "$SKIP_TORCH" = true ]; then else echo "==> Installing PyTorch ($TORCH_INDEX_URL)..." uv pip install --python "$_VENV_PY" "torch>=2.4,<2.11.0" torchvision torchaudio \ - --index-url "$TORCH_INDEX_URL" + --default-index "$TORCH_INDEX_URL" fi TORCH_EOF From cd9d251f157bc8a014a68f4688b961344d5d02f6 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 9 Jul 2026 04:10:59 -0700 Subject: [PATCH 019/372] Fix fast inference crash on compressed-tensors FP8 models (#7025) * Fix fast_gemv crash on compressed-tensors FP8 models Loading a compressed-tensors FP8 checkpoint (for example unsloth/Llama-3.2-1B-Instruct-FP8-Block) with fast_inference=False and running a forward crashed with 'Parameter object has no attribute absmax' inside fast_gemv. A compressed-tensors CompressedLinear exposes an already dequantized bf16 weight at forward time while keeping a weight_scale Parameter. The quant state resolution in get_lora_parameters/get_lora_parameters_bias fell back to that weight_scale, so a bf16 weight was routed into the bitsandbytes fast_gemv/fast_dequantize path, which expects a bitsandbytes QuantState with an absmax attribute. Only fall back to weight_scale_inv/weight_scale when the weight is still fp8. A decompressed bf16 weight then resolves to no quant state and flows through the normal bf16 path, which already handles bias and the LoRA backward. Real fp8 and bitsandbytes 4bit weights are unchanged. * Skip the fast_gemv dispatch test before importing unsloth when bitsandbytes is absent --- tests/test_fast_gemv_dispatch.py | 63 ++++++++++++++++++++++++++++++++ unsloth/kernels/utils.py | 27 ++++++++++++-- 2 files changed, 86 insertions(+), 4 deletions(-) create mode 100644 tests/test_fast_gemv_dispatch.py diff --git a/tests/test_fast_gemv_dispatch.py b/tests/test_fast_gemv_dispatch.py new file mode 100644 index 0000000000..7758db2cd9 --- /dev/null +++ b/tests/test_fast_gemv_dispatch.py @@ -0,0 +1,63 @@ +# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""`get_lora_parameters` must not treat a `weight_scale` as a quant state for a weight that is +already dequantized to bf16 (e.g. a compressed-tensors layer at forward time). Otherwise the +bnb fast_gemv / fast_dequantize path reads a missing `absmax` and crashes. +""" + +from types import SimpleNamespace + +import pytest +import torch + +# unsloth.kernels.utils imports bitsandbytes unconditionally, so skip the whole module up +# front on runners without it (e.g. CPU-only) before importing unsloth, otherwise collection +# errors instead of producing a skip. Any other import error still surfaces as a failure. +pytest.importorskip("bitsandbytes") + +import unsloth # noqa: F401 (sets UNSLOTH_IS_PRESENT before transformers) +from unsloth.kernels.utils import get_lora_parameters_bias, _FP8_WEIGHT_DTYPES + +_FP8 = _FP8_WEIGHT_DTYPES[0] if _FP8_WEIGHT_DTYPES else None + + +def _proj(weight, weight_scale = None): + proj = SimpleNamespace(weight = weight, bias = None, merged = False) + if weight_scale is not None: + proj.weight_scale = weight_scale + return proj + + +def test_bf16_weight_scale_not_used_as_quant_state(): + """A bf16 weight carrying a weight_scale (compressed-tensors) -> quant state must be None.""" + proj = _proj(torch.randn(4, 4, dtype = torch.bfloat16), torch.rand(2, 2)) + W, W_quant = get_lora_parameters_bias(proj)[:2] + assert W_quant is None + + +def test_fp8_weight_keeps_scale(): + """An actual fp8 weight still resolves its weight_scale as the quant state.""" + if _FP8 is None: + pytest.skip("no float8 dtype in this torch build") + scale = torch.rand(2, 2) + proj = _proj(torch.randn(4, 4).to(_FP8), scale) + W, W_quant = get_lora_parameters_bias(proj)[:2] + assert W_quant is scale + + +def test_plain_bf16_has_no_quant_state(): + proj = _proj(torch.randn(4, 4, dtype = torch.bfloat16)) + W, W_quant = get_lora_parameters_bias(proj)[:2] + assert W_quant is None diff --git a/unsloth/kernels/utils.py b/unsloth/kernels/utils.py index 43ed198a4a..1b0b5ce12e 100644 --- a/unsloth/kernels/utils.py +++ b/unsloth/kernels/utils.py @@ -282,6 +282,21 @@ def QUANT_STATE(W): return getattr(W, "quant_state", None) +# fp8 weight dtypes. A `weight_scale` / `weight_scale_inv` should only be treated as a +# quant state when the weight itself is still fp8. compressed-tensors layers expose an +# already-dequantized bf16 weight at forward time while keeping a `weight_scale` around; +# reading that as a quant state routes a bf16 weight into the bitsandbytes fast_gemv / +# fast_dequantize path, which then reads a missing `absmax` and crashes. +_FP8_WEIGHT_DTYPES = tuple( + dtype + for dtype in ( + getattr(torch, "float8_e4m3fn", None), + getattr(torch, "float8_e5m2", None), + ) + if dtype is not None +) + + def get_lora_parameters(proj): """Return (weight, weight quant_state, lora A, lora B, lora scale). With QAT enabled, also fake-quantizes the base layer and lora weights. @@ -298,9 +313,11 @@ def get_lora_parameters(proj): if weight_fake_quantizer is not None: W = weight_fake_quantizer(W) - # Get quant state for 4bit or FP8 + # Get quant state for 4bit or FP8. Only fall back to a weight_scale(_inv) when the + # weight is still fp8; a bf16 weight (e.g. a decompressed compressed-tensors layer) + # must not carry a scale as its quant state or fast_gemv will crash on it. W_quant = getattr(W, "quant_state", None) - if W_quant is None: + if W_quant is None and W.dtype in _FP8_WEIGHT_DTYPES: W_quant = getattr(base_layer, "weight_scale_inv", None) if W_quant is None: W_quant = getattr(base_layer, "weight_scale", None) @@ -349,9 +366,11 @@ def get_lora_parameters_bias(proj): ) # (proj.base_layer if hasattr(proj, "base_layer") else proj) W = base_layer.weight - # Get quant state for 4bit or FP8 + # Get quant state for 4bit or FP8. Only fall back to a weight_scale(_inv) when the + # weight is still fp8; a bf16 weight (e.g. a decompressed compressed-tensors layer) + # must not carry a scale as its quant state or fast_gemv will crash on it. W_quant = getattr(W, "quant_state", None) - if W_quant is None: + if W_quant is None and W.dtype in _FP8_WEIGHT_DTYPES: W_quant = getattr(base_layer, "weight_scale_inv", None) if W_quant is None: W_quant = getattr(base_layer, "weight_scale", None) From 534c877d2136b47b0ceec25cc45900c7f7f15e6d Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 9 Jul 2026 04:20:41 -0700 Subject: [PATCH 020/372] Keep native RoPE scaling when extending context; carry rope_theta for linear (#7028) * Keep native RoPE scaling when extending context; carry rope_theta for linear When max_seq_length exceeds a model's native window, the loader overwrote the model's rope_scaling with linear scaling. For models that already ship a scaled RoPE (llama3/yarn/longrope) that is far worse for long context, and on transformers v5 the linear dict omitted rope_theta (v5 keeps it under rope_parameters), so the rotary base fell back to 10000 and broke past ~8K tokens. Keep the native scaling and just widen the window; only synthesize linear for plain-RoPE models, and carry rope_theta so v5 keeps the real base. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Only preserve native llama3 when extending context; keep linear fallback otherwise The patched attention constructor (patch_llama_rope_scaling) rebuilds only linear, llama3 and longrope and its longrope branch reads a top-level original_max_position_embeddings, so preserving yarn or a nested-only longrope config would raise during construction on transformers <= 4.47.1. Keep only llama3 native; yarn/longrope/other types fall back to the linear override, still carrying rope_theta. * Correct long-context extension comment to match llama3-only preservation --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- tests/utils/test_rope_scaling_drift.py | 33 ++++++++++++++ unsloth/models/llama.py | 61 +++++++++++++++++--------- 2 files changed, 73 insertions(+), 21 deletions(-) diff --git a/tests/utils/test_rope_scaling_drift.py b/tests/utils/test_rope_scaling_drift.py index 7a738e236c..b2ec1e5a20 100644 --- a/tests/utils/test_rope_scaling_drift.py +++ b/tests/utils/test_rope_scaling_drift.py @@ -257,6 +257,39 @@ def test_recompute_helper_scales_on_cpu(): ), "_unsloth_recompute_inv_freq must return vanilla inv_freq when unscaled." +def test_extended_rope_scaling_keeps_llama3_and_carries_theta(): + # Long-context extension keeps native llama3, but falls back to linear for every other + # type (the patched attention constructor only rebuilds linear/llama3/longrope), and the + # linear dict carries rope_theta so transformers v5 does not fall back to base 10000. + from types import SimpleNamespace + + from unsloth.models.llama import _extended_rope_scaling + + # llama3 model: keep native scaling, do not synthesize linear. + scaling, native = _extended_rope_scaling(_make_config(LLAMA3_ROPE_SCALING), 2.0) + assert ( + scaling is None and native == "llama3" + ), "must keep native llama3 scaling instead of overwriting it with linear." + + # yarn is not rebuildable by the patcher -> keep the safe linear fallback, not native. + yarn = SimpleNamespace(rope_scaling = {"rope_type": "yarn", "factor": 2.0}, rope_theta = 500000.0) + scaling, _ = _extended_rope_scaling(yarn, 2.0) + assert scaling == { + "type": "linear", + "factor": 2.0, + "rope_theta": 500000.0, + }, f"yarn must fall back to linear (patcher cannot rebuild it), got {scaling}." + + # plain RoPE with theta only under v5 rope_parameters: linear must carry rope_theta. + v5 = SimpleNamespace(rope_parameters = {"rope_type": "default", "rope_theta": 1000000.0}) + scaling, _ = _extended_rope_scaling(v5, 2.0) + assert scaling == { + "type": "linear", + "factor": 2.0, + "rope_theta": 1000000.0, + }, f"linear override dropped rope_theta on v5 (got {scaling}); base would fall back to 10000." + + def test_extended_rotary_reads_config_factor(): # LlamaExtendedRotaryEmbedding must honor the config factor, not hardcode 8 # (Llama-3.2 uses 32); otherwise the subclass path re-drops scaling (#2405). diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index 1f43f61443..05523bc27b 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -1651,6 +1651,26 @@ def _rope_scaling_as_dict(rope_scaling): return {} +def _extended_rope_scaling(config, factor): + """RoPE scaling to extend a model past its native window. Keeps native llama3 as-is + (linear extension is far worse for long context); everything else gets linear. Returns + (scaling_or_None, type): None keeps llama3. The linear dict carries rope_theta so + transformers v5 (which stores it under rope_parameters) keeps the real base, not 10000. + Only llama3 is preserved because patch_llama_rope_scaling can only rebuild linear/llama3/ + longrope and its longrope branch needs a top-level original_max_position_embeddings.""" + existing = _rope_scaling_as_dict( + getattr(config, "rope_scaling", None) or getattr(config, "rope_parameters", None) or {} + ) + existing_type = existing.get("rope_type") or existing.get("type") + if existing_type == "llama3": + return None, existing_type + return { + "type": "linear", + "factor": factor, + "rope_theta": _get_rope_theta(config), + }, existing_type + + def _llama3_inv_freq_from_config( config, rope_scaling, @@ -2518,34 +2538,33 @@ class FastLlamaModel: max_seq_length = model_max_seq_length if (rope_scaling is None) and (max_seq_length > model_max_seq_length): - rope_scaling = max_seq_length / model_max_seq_length + factor = max_seq_length / model_max_seq_length if fast_inference: raise NotImplementedError( "Unsloth: Fast inference does not yet work with RoPE Scaling." ) - logger.warning_once( - f"Unsloth: {model_name} can only handle sequence lengths of at most " - f"{model_max_seq_length}.\nBut with kaiokendev's RoPE scaling of " - f"{round(rope_scaling, 3)}, it can be magically be extended to " - f"{max_seq_length}!" - ) - - # Warn RoPE scaling isn't allowed - if not has_rope_scaling: - raise RuntimeError( - f"However, {model_name} doesn't support RoPE Scaling!\n" - "Please file a feature request at https://github.com/unslothai/unsloth." + linear_scaling, native_type = _extended_rope_scaling(model_config, factor) + if linear_scaling is not None: + logger.warning_once( + f"Unsloth: {model_name} can only handle sequence lengths of at most " + f"{model_max_seq_length}.\nBut with kaiokendev's RoPE scaling of " + f"{round(factor, 3)}, it can be magically be extended to " + f"{max_seq_length}!" + ) + if not has_rope_scaling: + raise RuntimeError( + f"However, {model_name} doesn't support RoPE Scaling!\n" + "Please file a feature request at https://github.com/unslothai/unsloth." + ) + kwargs["rope_scaling"] = linear_scaling + else: + # Native llama3 scaling already handles long context; just widen the window. + logger.warning_once( + f"Unsloth: extending {model_name} to {max_seq_length} using its native " + f"{native_type} RoPE scaling." ) - - rope_scaling = { - "type": "linear", - "factor": rope_scaling, - } - - # Add to kwargs - kwargs["rope_scaling"] = rope_scaling from .loader_utils import ( check_and_disable_bitsandbytes_loading, From b5dca66cb1480b36ef738a4e62680e02cca2a65f Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 9 Jul 2026 04:52:30 -0700 Subject: [PATCH 021/372] scripts: refresh scan_packages allowlist baseline (#7032) * scripts: refresh scan_packages allowlist baseline Regenerate scripts/scan_packages_baseline.json against the current resolved dependency set so the blocking pip scan-packages gate matches what the scanner now finds. Refreshes evidence hashes for benign findings whose code shifted lines (unsloth-zoo mlx loader, gguf/mlx test /tmp fixtures) and adds two mainstream-library entries that were newly surfaced (torch inductor codecache base64+subprocess compile cache, torch testing common_utils socket import). Stale entries whose matching code changed and no longer triggers are dropped. All entries remain CRITICAL/HIGH findings manually judged benign; matched on (package, file, check, evidence_hash). * ci(security-audit): re-run scan when the allowlist baseline changes The security-audit pull_request trigger listed the scanners but not their allowlist baselines, so a baseline-only edit never re-ran the scan that consumes it. A refreshed baseline could therefore merge without CI confirming its evidence hashes match what the scanner finds. Add scan_packages_baseline.json and scan_npm_packages_baseline.json to the paths filter so baseline changes are validated on their own PR. --- .github/workflows/security-audit.yml | 6 +- scripts/scan_packages_baseline.json | 304 ++++++++++++--------------- 2 files changed, 140 insertions(+), 170 deletions(-) diff --git a/.github/workflows/security-audit.yml b/.github/workflows/security-audit.yml index 0ef2ad1e9d..1275d12216 100644 --- a/.github/workflows/security-audit.yml +++ b/.github/workflows/security-audit.yml @@ -2,8 +2,8 @@ # Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. # Multi-language supply-chain audit. Triggers: -# - PRs touching any dependency manifest (Python / npm / Cargo) or -# this workflow file, +# - PRs touching any dependency manifest (Python / npm / Cargo), a +# scanner or its allowlist baseline, or this workflow file, # - push to main / pip, # - nightly @ 04:13 UTC so newly-published advisories surface even # when no PR opens, @@ -57,7 +57,9 @@ on: - 'studio/src-tauri/Cargo.lock' - 'pyproject.toml' - 'scripts/scan_packages.py' + - 'scripts/scan_packages_baseline.json' - 'scripts/scan_npm_packages.py' + - 'scripts/scan_npm_packages_baseline.json' - '.github/workflows/security-audit.yml' push: branches: [main, pip] diff --git a/scripts/scan_packages_baseline.json b/scripts/scan_packages_baseline.json index 1d34cfb66d..3582517d31 100644 --- a/scripts/scan_packages_baseline.json +++ b/scripts/scan_packages_baseline.json @@ -39,7 +39,7 @@ "file": "botocore/utils.py", "check": "Reads credential paths AND makes network calls", "severity": "CRITICAL", - "evidence": "Creds: L3551: CACHE_DIR = os.path.expanduser(os.path.join('~', '.aws', 'boto', 'cache')) | L3721: return os.path.expanduser(os.path.join('~', '.aws', 'login', 'cache'))\nNetwork: L32: from urllib.request import getproxies, proxy_bypass", + "evidence": "Creds: L3551: CACHE_DIR = os.path.expanduser(os.path.join('~', '.aws', 'boto', 'cache')) | L3719: return os.path.expanduser(os.path.join('~', '.aws', 'login', 'cache'))\nNetwork: L32: from urllib.request import getproxies, proxy_bypass", "evidence_hash": "2d691bc373ab872aad23c744104596ba6d0d9f3b35aa101c7edbff4429b174c1" }, { @@ -55,23 +55,23 @@ "file": "datasets/utils/file_utils.py", "check": "C2 polling/beaconing loop detected", "severity": "CRITICAL", - "evidence": "L443: while True: sha256:feba37d77721aa658e1786d2e4b67de76fefe1ceeb3ce8529d361c5241778eea", - "evidence_hash": "2e458563dec752d0a9896c9685d368d9906867110db315ab751e3eb6ec63f51c" + "evidence": "L441: while True: sha256:ce92e38c17c524815e1f9055be77235028c1e68e41b45cbfe9c8f1b867a205da", + "evidence_hash": "cb36281d28a975d101121c0702ee05eeee470879520d39a8be552129333f514d" }, { "package": "datasets", "file": "datasets/utils/file_utils.py", "check": "C2 polling/beaconing loop detected", "severity": "CRITICAL", - "evidence": "L441: while True: sha256:ce92e38c17c524815e1f9055be77235028c1e68e41b45cbfe9c8f1b867a205da", - "evidence_hash": "cb36281d28a975d101121c0702ee05eeee470879520d39a8be552129333f514d" + "evidence": "L443: while True: sha256:feba37d77721aa658e1786d2e4b67de76fefe1ceeb3ce8529d361c5241778eea", + "evidence_hash": "2e458563dec752d0a9896c9685d368d9906867110db315ab751e3eb6ec63f51c" }, { "package": "diffusers", "file": "diffusers/utils/import_utils.py", "check": "Downloads and executes remote code", "severity": "CRITICAL", - "evidence": "L1015: return importlib.import_module(\".\" + module_name, self.__name__)", + "evidence": "L1052: return importlib.import_module(\".\" + module_name, self.__name__)", "evidence_hash": "e584ecfdb097d9482bb19cd3992813bc1a119cfd4c40af14748bafe22900d91e" }, { @@ -79,7 +79,7 @@ "file": "diffusers/utils/testing_utils.py", "check": "Harvests environment variables/secrets AND makes network calls", "severity": "CRITICAL", - "evidence": "Env: L233: value = os.environ[key]\nNetwork: L688: response = requests.get(arry, timeout=DIFFUSERS_REQUEST_TIMEOUT) | L709: response = requests.get(url, timeout=DIFFUSERS_REQUEST_TIMEOUT) | L728: image = PIL.Image.open(requests.get(image, stream=True, timeout=DIFFUSERS_REQUEST_TIMEOUT).raw)", + "evidence": "Env: L236: value = os.environ[key]\nNetwork: L691: response = requests.get(arry, timeout=DIFFUSERS_REQUEST_TIMEOUT) | L712: response = requests.get(url, timeout=DIFFUSERS_REQUEST_TIMEOUT) | L731: image = PIL.Image.open(requests.get(image, stream=True, timeout=DIFFUSERS_REQUEST_TIMEOUT).raw)", "evidence_hash": "671190a6106c6ee9674e5e5942dc0940e1d2f8c78d5faf674413c2345b783fd9" }, { @@ -90,12 +90,20 @@ "evidence": "Archive: L317: a['TarFileType'] = tarfile.open(fileobj=_fileW,mode='w')\nNetwork: L330: x['SocketType'] = _socket = socket.socket()", "evidence_hash": "894862e547cf91b90cd6e4b495db3fb05b7490ef0d63de7e795a7e3d9447d850" }, + { + "package": "fastapi", + "file": "fastapi/routing.py", + "check": "C2 polling/beaconing loop detected", + "severity": "CRITICAL", + "evidence": "L586: while True: sha256:251135b5ebfdd1248916449f32262575e003ef64382501c65b7e4061d67bda45", + "evidence_hash": "365aef4449c8089753d9398417cd76ab762cef547d75db70d87bca9c0b550ab5" + }, { "package": "fastmcp-slim", "file": "fastmcp/cli/apps_dev.py", "check": "Creates archive with sensitive data AND makes network calls", "severity": "CRITICAL", - "evidence": "Archive: L1340: with tarfile.open(fileobj=io.BytesIO(data), mode=\"r:gz\") as tar:\nNetwork: L1291: with httpx.Client(timeout=30.0) as client: | L1305: with httpx.Client(timeout=30.0) as client: | L1335: with httpx.Client(timeout=30.0) as client: | L1537: client = httpx.AsyncClient(\nL1538: timeout=httpx.Timeout(60.0, read=None), trust_env=False\nL1539: ) | L1701: async with httpx.AsyncClient(trust_env=False) as client: | L1769: with socket.socket(family, socket.SOCK_STREAM) as s:", + "evidence": "Archive: L1353: with tarfile.open(fileobj=io.BytesIO(data), mode=\"r:gz\") as tar:\nNetwork: L1304: with httpx.Client(timeout=30.0) as client: | L1318: with httpx.Client(timeout=30.0) as client: | L1348: with httpx.Client(timeout=30.0) as client: | L1549: client = httpx.AsyncClient(\nL1550: timeout=httpx.Timeout(60.0, read=None), trust_env=False\nL1551: ) | L1713: async with httpx.AsyncClient(trust_env=False) as client: | L1781: with socket.socket(family, socket.SOCK_STREAM) as s:", "evidence_hash": "73a7a72013e9f800627ea07e6dbc3beeb8c905a6a5480c8fd896f0063173d25c" }, { @@ -103,8 +111,8 @@ "file": "fastmcp/cli/apps_dev.py", "check": "Enumerates filesystem AND makes network calls", "severity": "CRITICAL", - "evidence": "FS: L624: history.replaceState(null, \"\", url); sha256:fd8dbfa8af4dea2ce43f4d441f3f81239de341b76a2eb0a33c446f6757ce5f43\nNetwork: L1291: with httpx.Client(timeout=30.0) as client: | L1305: with httpx.Client(timeout=30.0) as client: | L1335: with httpx.Client(timeout=30.0) as client: | L1537: client = httpx.AsyncClient(\nL1538: timeout=httpx.Timeout(60.0, read=None), trust_env=False\nL1539: ) | L1701: async with httpx.AsyncClient(trust_env=False) as client: | L1769: with socket.socket(family, socket.SOCK_STREAM) as s:", - "evidence_hash": "6ada4a9111213bdee5ea24c70a72ec4acdc8ffe0de4a01fd9835bc261ccab8f8" + "evidence": "FS: L637: history.replaceState(null, \"\", url); sha256:17068ba5bfed62c3a3007ec8bf3e0ea41ef6529b9e6112064d9afb3be9231436\nNetwork: L1304: with httpx.Client(timeout=30.0) as client: | L1318: with httpx.Client(timeout=30.0) as client: | L1348: with httpx.Client(timeout=30.0) as client: | L1549: client = httpx.AsyncClient(\nL1550: timeout=httpx.Timeout(60.0, read=None), trust_env=False\nL1551: ) | L1713: async with httpx.AsyncClient(trust_env=False) as client: | L1781: with socket.socket(family, socket.SOCK_STREAM) as s:", + "evidence_hash": "e5325edfada6499540e6f0c24a0868979d275522e2b6a180aa9b5dd3280681b4" }, { "package": "fonttools", @@ -132,19 +140,35 @@ }, { "package": "huggingface-hub", - "file": "huggingface_hub/hf_api.py", + "file": "huggingface_hub/_sandbox.py", "check": "C2 polling/beaconing loop detected", "severity": "CRITICAL", - "evidence": "L3746: while True: sha256:0c73ed1a7447120b112c063b14e720c6695bc11d00eb6b912cd0f10dc3e29b31", - "evidence_hash": "22f50b930e44146c5350bb99e6e6ebb09feea9bf1e899e407bedc4ffaf06721b" + "evidence": "L1179: while True: sha256:33ceddf9e42aae207e891e97808c518e92a0b27ab60e4326256717bfb25a3a38", + "evidence_hash": "802fd41d8bb17bf425e99d128c0351c820103a5efb74690a4086e542a71437b8" + }, + { + "package": "huggingface-hub", + "file": "huggingface_hub/_sandbox.py", + "check": "Writes to /tmp and executes (staged dropper)", + "severity": "CRITICAL", + "evidence": "L83: d=/tmp/.sbx-server\nL84: if command -v wget >/dev/null 2>&1; then wget -q --header \"Authorization: Bearer $SBX_DL_TOKEN\" -O \"$d\" \"$SBX_SERVER_URL\"\nL85: elif command -v curl >/dev/null 2>&1; then curl -fsSL -H \"Authorization: Bearer $SBX_DL_TOKEN\" -o \"$d\" \"$SBX_SERVER_URL\"\nL86: else cp \"$SBX_SERVER_MOUNT/sbx-server\" \"$d\"; fi\nL87: chmod +x \"$d\"", + "evidence_hash": "6908a3fe328fa94ee22a119998d6ad07cfa1ba4efa2628acf240f4204fd76e22" }, { "package": "huggingface-hub", "file": "huggingface_hub/hf_api.py", "check": "C2 polling/beaconing loop detected", "severity": "CRITICAL", - "evidence": "L4600: while True: sha256:f4a851312a1832efe1b435aa1275a82184e19cc3f47e2cd244373d56c11de272", - "evidence_hash": "dc8fcf44788e32f42d1cc2eb0e2deb55eb2dbf2c3a55909a7d503e450f45e602" + "evidence": "L4613: while True: sha256:f764b6ca3118b23c7c0e670e77178c022a6905f825d7df6e528545fa10aae8f6", + "evidence_hash": "9c85d50c227285fa8dc69512999cbb082258cda4b299c7d0e0f69f5aff7accd4" + }, + { + "package": "huggingface-hub", + "file": "huggingface_hub/hf_api.py", + "check": "C2 polling/beaconing loop detected", + "severity": "CRITICAL", + "evidence": "L3746: while True: sha256:0c73ed1a7447120b112c063b14e720c6695bc11d00eb6b912cd0f10dc3e29b31", + "evidence_hash": "22f50b930e44146c5350bb99e6e6ebb09feea9bf1e899e407bedc4ffaf06721b" }, { "package": "huggingface-hub", @@ -159,8 +183,8 @@ "file": "huggingface_hub/utils/_http.py", "check": "C2 polling/beaconing loop detected", "severity": "CRITICAL", - "evidence": "L443: while True: sha256:0ab4fed32d3af10f361963371f681923481377508a405b5d8770cef75f859168", - "evidence_hash": "1484f6b92f41c427ba8cbc7c4695a94975fea683dfa83aa510b4b0e982be4721" + "evidence": "L462: while True: sha256:c75d1ee228cf7703a8c28551d649395a1f89f69a3aba69413f5bbcbd10c31958", + "evidence_hash": "d4d5f83fed39b87898cf776d5dad0bf1a6388a932f5fb7997d1070b50e46213e" }, { "package": "huggingface-hub", @@ -218,6 +242,22 @@ "evidence": "L5: import socket sha256:915068303029fa5806199f256fb74504c65f253f9aee8ea23d8e384bb772b1c7", "evidence_hash": "30be130f165f418dfd37b144c5ae333de184b95f828ab8bd4010a67b84a5f814" }, + { + "package": "multiprocess", + "file": "multiprocess/tests/__init__.py", + "check": "Reverse shell / bind shell pattern", + "severity": "CRITICAL", + "evidence": "L3355: os.dup2(conn.fileno(), i) | L3387: \"test needs os.dup2()\") | L3405: os.dup2(fd, newfd) | L19: import socket sha256:26a745abdc7e89da28ab943394234d8ccb415e805477c3cc1f7d4766341a4c4c", + "evidence_hash": "a6b9bb85e9bb6682ab0dea4f95fd9266e8802f118c76d86dd87f7ab5864872cf" + }, + { + "package": "multiprocess", + "file": "multiprocess/tests/__init__.py", + "check": "Reverse shell / bind shell pattern", + "severity": "CRITICAL", + "evidence": "L3521: os.dup2(conn.fileno(), i) | L3553: \"test needs os.dup2()\") | L3571: os.dup2(fd, newfd) | L20: import socket sha256:07d2933301c0dbeeb6e42381687827d8dd7cfd7471986c559ca64283d5ae6e24", + "evidence_hash": "db1f4ca69865ec3911d7450fe11d212b817139deda21cd7a4ee32d547a8dc452" + }, { "package": "numba", "file": "numba/pycc/decorators.py", @@ -231,7 +271,7 @@ "file": "numba/tests/support.py", "check": "Reverse shell / bind shell pattern", "severity": "CRITICAL", - "evidence": "L1021: os.dup2(w, fd) | L1026: os.dup2(save, fd)", + "evidence": "L1016: os.dup2(w, fd) | L1021: os.dup2(save, fd)", "evidence_hash": "fea7aa03d48bf0f4386302fa444984c4f5dfc772cfec3f1df199fd33a52eec10" }, { @@ -495,16 +535,16 @@ "file": "sklearn/datasets/_openml.py", "check": "C2 polling/beaconing loop detected", "severity": "CRITICAL", - "evidence": "L100: while True: sha256:270363bb66980201e477f9b94886e4023f7a3d21b5ce026b7603a8c249a50c5b", - "evidence_hash": "53edbe07c312d459068d38e537b5114e65685ac3d4487b0423fa4542b5df20fe" + "evidence": "L100: while True: sha256:1f05a1b4fdd843b309634f583cb5e919866ef38ec5aa0b7d8a66ac8820655594", + "evidence_hash": "69597a64e5670a0f9a3c2aafc0bde4160f6170a9e2dc38f2c413cfa8d22ad193" }, { "package": "scikit-learn", "file": "sklearn/datasets/_openml.py", "check": "C2 polling/beaconing loop detected", "severity": "CRITICAL", - "evidence": "L100: while True: sha256:1f05a1b4fdd843b309634f583cb5e919866ef38ec5aa0b7d8a66ac8820655594", - "evidence_hash": "69597a64e5670a0f9a3c2aafc0bde4160f6170a9e2dc38f2c413cfa8d22ad193" + "evidence": "L100: while True: sha256:270363bb66980201e477f9b94886e4023f7a3d21b5ce026b7603a8c249a50c5b", + "evidence_hash": "53edbe07c312d459068d38e537b5114e65685ac3d4487b0423fa4542b5df20fe" }, { "package": "scikit-learn", @@ -642,6 +682,14 @@ "evidence": "Base64: L1211: content = base64.b64decode(data)\nSubprocess: L2692: subprocess.run(\nL2693: cmd.split(), capture_output=True, text=True, check=True\nL2694: ) | L2995: cmd_output = subprocess.run(\nL2996: (\"openssl\", \"sha512\", filename), capture_output=True, text=True\nL2997: ) | L3707: out = subprocess.check_output(\nL3708: [\"ldd\", os.path.join(search, file)]\nL3709: ) | L3791: jobs.append(functools.partial(subprocess.check_call, cmd)) | L3876: subprocess.check_call(\nL3877: shlex.split(halide_cmd_gen.get_command_line())\nL3878: ) | L4336: subprocess.check_output(\nL4337: cmd_parts, stderr=subprocess.STDOUT, env=os.environ\nL4338: ) | L4591: output = subprocess.check_output(\nL4592: cmd_parts,\nL4593: stderr=subprocess.STDOUT,\nL4594: text=True,\nL4595: env=os.environ,\nL4596: )", "evidence_hash": "c09774087b702a6c5d6e2e85d9239c7c241ec938fbe9c0153e8f0b5c0710389b" }, + { + "package": "torch", + "file": "torch/_inductor/codecache.py", + "check": "base64 decode + subprocess execution (staged payload)", + "severity": "CRITICAL", + "evidence": "Base64: L1727: content = base64.b64decode(data)\nSubprocess: L3270: subprocess.run(\nL3271: cmd, capture_output=True, text=True, check=True\nL3272: ) | L3583: cmd_output = subprocess.run(\nL3584: (\"openssl\", \"sha512\", filename), capture_output=True, text=True\nL3585: ) | L4338: out = subprocess.check_output(\nL4339: [\"ldd\", os.path.join(search, file)]\nL4340: ) | L4422: jobs.append(functools.partial(subprocess.check_call, cmd)) | L4507: subprocess.check_call(\nL4508: shlex.split(halide_cmd_gen.get_command_line())\nL4509: ) | L4992: subprocess.check_output(\nL4993: cmd_parts, stderr=subprocess.STDOUT, env=os.environ\nL4994: ) | L5247: output = subprocess.check_output(\nL5248: cmd_parts,\nL5249: stderr=subprocess.STDOUT,\nL5250: text=True,\nL5251: env=os.environ,\nL5252: )", + "evidence_hash": "87f77b5f51cb84fe9950fdeeb90fe8710e1b863100e90b5e2cfb228a725bee06" + }, { "package": "torch", "file": "torch/ao/__init__.py", @@ -695,7 +743,7 @@ "file": "torch/testing/_internal/common_utils.py", "check": "Harvests environment variables/secrets AND makes network calls", "severity": "CRITICAL", - "evidence": "Env: L4770: env = os.environ.copy()\nNetwork: L4832: with request.urlopen(url, timeout=15) as f1, open(path, 'wb' if binary else 'w') as f2: | L4850: with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock:", + "evidence": "Env: L4900: env = os.environ.copy()\nNetwork: L4962: with request.urlopen(url, timeout=15) as f1, open(path, 'wb' if binary else 'w') as f2: | L4980: with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock:", "evidence_hash": "704a851b9d68c9b885b9e15538bd7e96f03875503b618fe6f126c4438edd7386" }, { @@ -706,6 +754,14 @@ "evidence": "L32: import socket sha256:89faaaa8bc908e02dad73fd59b2b481fa91189c84b39b556c2766e71d2783bf3", "evidence_hash": "3d23d77ace91812a07cb9508cf352185d154176e8e8c8b9b28fa92cdbcfe0d53" }, + { + "package": "torch", + "file": "torch/testing/_internal/common_utils.py", + "check": "Reverse shell / bind shell pattern", + "severity": "CRITICAL", + "evidence": "L32: import socket sha256:ba439cbf568b194872f1d974c02b0487e51f677b67e379400522d0992600bd2d", + "evidence_hash": "88e98b227573997f86eedea8e885a407b0dd549d46d4a3f0b840ec5aafe66865" + }, { "package": "torchvision", "file": "torchvision/datasets/utils.py", @@ -743,8 +799,8 @@ "file": "transformers/testing_utils.py", "check": "C2 polling/beaconing loop detected", "severity": "CRITICAL", - "evidence": "L1663: while True: sha256:969e911d30c37a279ad915fb8c3d2d0a3f5705a7eb82ae6e00687388b68bbe65", - "evidence_hash": "2aa8e94baa805d599720a16afee6f08976482e301333e619e6c343389498ad15" + "evidence": "L1577: while True: sha256:2c6152f9da685f728e58d39dfc1827bc794f52606f56983bf38b5c6d0857cd5b", + "evidence_hash": "cdada67f3327237f00838a6750a4908dfaf76b9ab30c1352495c340d4fbd15c9" }, { "package": "transformers", @@ -759,15 +815,15 @@ "file": "transformers/testing_utils.py", "check": "C2 polling/beaconing loop detected", "severity": "CRITICAL", - "evidence": "L1577: while True: sha256:2c6152f9da685f728e58d39dfc1827bc794f52606f56983bf38b5c6d0857cd5b", - "evidence_hash": "cdada67f3327237f00838a6750a4908dfaf76b9ab30c1352495c340d4fbd15c9" + "evidence": "L1699: while True: sha256:969e911d30c37a279ad915fb8c3d2d0a3f5705a7eb82ae6e00687388b68bbe65", + "evidence_hash": "2aa8e94baa805d599720a16afee6f08976482e301333e619e6c343389498ad15" }, { "package": "transformers", "file": "transformers/testing_utils.py", "check": "Harvests environment variables/secrets AND makes network calls", "severity": "CRITICAL", - "evidence": "Env: L284: value = os.environ[key] | L300: value = os.environ[key] | L2129: env = os.environ.copy() | L2251: for k in list(os.environ.keys()):\nNetwork: L2561: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:", + "evidence": "Env: L288: value = os.environ[key] | L304: value = os.environ[key] | L2165: env = os.environ.copy() | L2287: for k in list(os.environ.keys()):\nNetwork: L2597: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:", "evidence_hash": "73ff16aee09cf163fb3a7a04dfa2cf610595bde2f19460a579397695f728e3f4" }, { @@ -799,16 +855,16 @@ "file": "trl/extras/vllm_client.py", "check": "C2 polling/beaconing loop detected", "severity": "CRITICAL", - "evidence": "L146: while True: sha256:2beedc742e1f085eaa10fd3bc40be97d2331d21887ef1b9ccdfa2150a184edfe", - "evidence_hash": "1540dffaaa053780e953e04c11d9c6b9c74b91cb60f3e6d87451ba7fe7db46db" + "evidence": "L152: while True: sha256:93e7d409e300af445376e6defbe2d0241aa19ecf63ed41b780fbb91c7d09856f", + "evidence_hash": "208838617172de61bca201d2a1bbeb5aa5aaa55feb1a1069cf39214673a7d6d1" }, { "package": "trl", "file": "trl/extras/vllm_client.py", "check": "C2 polling/beaconing loop detected", "severity": "CRITICAL", - "evidence": "L152: while True: sha256:93e7d409e300af445376e6defbe2d0241aa19ecf63ed41b780fbb91c7d09856f", - "evidence_hash": "208838617172de61bca201d2a1bbeb5aa5aaa55feb1a1069cf39214673a7d6d1" + "evidence": "L146: while True: sha256:2beedc742e1f085eaa10fd3bc40be97d2331d21887ef1b9ccdfa2150a184edfe", + "evidence_hash": "1540dffaaa053780e953e04c11d9c6b9c74b91cb60f3e6d87451ba7fe7db46db" }, { "package": "trl", @@ -866,6 +922,14 @@ "evidence": "Crypto: L294: r\"|\\b(?:xprv|xpub|bc1|0x[a-fA-F0-9]{40})\\b\",\nNetwork: L53: import urllib.request | L1757: req = urllib.request.Request(url, headers = {\"Accept\": \"application/json\"}) | L1758: with urllib.request.urlopen(req, timeout = 30) as resp:", "evidence_hash": "278ff15b0b702d37d7f0b30a1e55a31bf2b11883685718a47478fbb5ce7f5212" }, + { + "package": "unsloth-zoo", + "file": "scripts/scan_packages.py", + "check": "Writes to /tmp and executes (staged dropper)", + "severity": "CRITICAL", + "evidence": "L308: r\"/tmp/\\S+.*(?:subprocess|os\\.system|os\\.popen|Popen|chmod.*\\+x)\", | L308: r\"/tmp/\\S+.*(?:subprocess|os\\.system|os\\.popen|Popen|chmod.*\\+x)\", sha256:78268349021e21bedcd2eaaa5b4a71b0de1d52e023ada914dfdc09515ee1aad8", + "evidence_hash": "590fe1c96c442fbea5eb8642650257bc0b0199e919b9bacdb11dfa767b6fe839" + }, { "package": "unsloth-zoo", "file": "tests/security/fixtures/_build.py", @@ -919,16 +983,16 @@ "file": "tests/test_mlx_save_export_regressions.py", "check": "Writes to /tmp and executes (staged dropper)", "severity": "CRITICAL", - "evidence": "L164: temporary_location=\"/tmp/ignored\", sha256:78837e80d48e872ef191aaacfe5e1c621a98a20df486a70a41d1a932d074a5b3", - "evidence_hash": "dd11376e664d0d7e7f4cc4baf57eacd4b7ae7b03222dce3912ce68b63dbfca1e" + "evidence": "L165: temporary_location=\"/tmp/ignored\", sha256:9f8502377b19666288b28399633dfc6740a64d0cb70ad1615e38b1269f94bf37", + "evidence_hash": "b7262d6e58f2ebad961dd3e64ca6c32bba356b5044d7a642d7dbd36a58cb6c81" }, { "package": "unsloth-zoo", "file": "tests/test_quantize_gguf_q2_k_l.py", "check": "Writes to /tmp and executes (staged dropper)", "severity": "CRITICAL", - "evidence": "L67: input_gguf=\"/tmp/in.gguf\", sha256:32532cadc357beee1009f4e86481bdbe60a0b7bf47f6bb022b05ec1b8e15aed0", - "evidence_hash": "49f5b67379de17178f21a9bc93b79d6b94a70ecbdd16de86574934aac30a071d" + "evidence": "L67: input_gguf=\"/tmp/in.gguf\", sha256:06789b55e8f31426c233f37ff7d3729cc9e1f61c0829abd2c00c39216c63c7ad", + "evidence_hash": "ad4913d9099eb9b70e09d6860b242eb5f48c67e46d9bf4ae35c1c38a267d753b" }, { "package": "unsloth-zoo", @@ -951,7 +1015,7 @@ "file": "unsloth_zoo/llama_cpp.py", "check": "Creates archive with sensitive data AND makes network calls", "severity": "CRITICAL", - "evidence": "Archive: L938: with tarfile.open(archive_path, \"r:gz\") as archive:\nNetwork: L691: response = requests.get(url, timeout = timeout, headers = headers, stream = stream) | L1699: response = requests.get(\nL1700: LLAMA_CPP_CONVERT_FILE, timeout = (10, 120)\nL1701: ) | L2862: check = requests.get(llama_cpp_chat_file, timeout = 5)", + "evidence": "Archive: L938: with tarfile.open(archive_path, \"r:gz\") as archive:\nNetwork: L691: response = requests.get(url, timeout = timeout, headers = headers, stream = stream) | L1699: response = requests.get(\nL1700: LLAMA_CPP_CONVERT_FILE, timeout = (10, 120)\nL1701: ) | L2873: check = requests.get(llama_cpp_chat_file, timeout = 5)", "evidence_hash": "b9f3b1652349fa8ef9ac2d1715978aca1e1632165851a00a2698dd47189e410c" }, { @@ -959,7 +1023,7 @@ "file": "unsloth_zoo/llama_cpp.py", "check": "Harvests environment variables/secrets AND makes network calls", "severity": "CRITICAL", - "evidence": "Env: L125: keynames = \"\\n\" + \"\\n\".join(os.environ.keys()) | L683: token = os.environ.get(\"GH_TOKEN\") or os.environ.get(\"GITHUB_TOKEN\")\nNetwork: L691: response = requests.get(url, timeout = timeout, headers = headers, stream = stream) | L1699: response = requests.get(\nL1700: LLAMA_CPP_CONVERT_FILE, timeout = (10, 120)\nL1701: ) | L2862: check = requests.get(llama_cpp_chat_file, timeout = 5)", + "evidence": "Env: L125: keynames = \"\\n\" + \"\\n\".join(os.environ.keys()) | L683: token = os.environ.get(\"GH_TOKEN\") or os.environ.get(\"GITHUB_TOKEN\")\nNetwork: L691: response = requests.get(url, timeout = timeout, headers = headers, stream = stream) | L1699: response = requests.get(\nL1700: LLAMA_CPP_CONVERT_FILE, timeout = (10, 120)\nL1701: ) | L2873: check = requests.get(llama_cpp_chat_file, timeout = 5)", "evidence_hash": "9cd0b1bb59c7eb1d814d7636dfd167c34f265eb7c4521a9d88b2bdcfd535b926" }, { @@ -1002,6 +1066,14 @@ "evidence": "Obfusc: L87: __import__(name)\nExec: L735: exec(\"\"\"exec _code_ in _globs_, _locs_\"\"\")", "evidence_hash": "3cb7d8247dea7dd3d7b21ededc0181c58c50099aeb73c9138a286f3d1ad92d4f" }, + { + "package": "cffi", + "file": "cffi/_cffi_gen_src.py", + "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", + "severity": "HIGH", + "evidence": "Obfusc: L52: compiled = compile(source=pysrc, filename=filename, mode='exec')\nExec: L53: exec(compiled, globs, globs)", + "evidence_hash": "c429e4c977a61db6b7c717b5a552fce74eda622213e49eb5467a3782fd746fb9" + }, { "package": "cffi", "file": "cffi/setuptools_ext.py", @@ -1127,7 +1199,7 @@ "file": "numba/tests/support.py", "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", "severity": "HIGH", - "evidence": "Obfusc: L879: __import__(modname)\nExec: L813: eval(co, globs, ns)", + "evidence": "Obfusc: L874: __import__(modname)\nExec: L808: eval(co, globs, ns)", "evidence_hash": "649a7d750f903478243b0bcb9e8020521b505fc7fedc5b696ec01f4efc096109" }, { @@ -1159,7 +1231,7 @@ "file": "numba/tests/test_np_functions.py", "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", "severity": "HIGH", - "evidence": "Obfusc: L7106: exec(compile(funcstr, '', 'exec'), globals(), dct)\nExec: L7106: exec(compile(funcstr, '', 'exec'), globals(), dct)", + "evidence": "Obfusc: L7118: exec(compile(funcstr, '', 'exec'), globals(), dct)\nExec: L7118: exec(compile(funcstr, '', 'exec'), globals(), dct)", "evidence_hash": "9e81164131d16056fb56ad3cd11b8d129d1ff4f5855031e8b501e0335d5c14ed" }, { @@ -1175,16 +1247,16 @@ "file": "numpy/testing/_private/utils.py", "check": "Anti-analysis/sandbox evasion + suspicious behavior", "severity": "HIGH", - "evidence": "Anti: L2777: original_trace = sys.gettrace() | L2779: sys.settrace(None) | L2782: sys.settrace(original_trace)\nSubprocess: L1478: output = subprocess.run(cmd, capture_output=True, text=True)\nExec: L1346: exec(astr, dict) | L1632: exec(code, globs, locs)", - "evidence_hash": "27468a6828101c6c026ae25aca8aa90ef485fd62b2c8f0967479edae9c965844" + "evidence": "Anti: L2788: original_trace = sys.gettrace() | L2790: sys.settrace(None) | L2793: sys.settrace(original_trace)\nSubprocess: L1486: output = subprocess.run(cmd, capture_output=True, text=True) | L2889: res = subprocess.run(cmd, cwd=cwd, capture_output=True, text=True,\nL2890: errors=\"replace\", **kwargs)\nExec: L1352: exec(astr, dict) | L1640: exec(code, globs, locs)", + "evidence_hash": "9c6961817e5b1751e572dfe0858286703bb835870ecdfd6a7a9fdd8372a5dd2b" }, { "package": "numpy", "file": "numpy/testing/_private/utils.py", "check": "Anti-analysis/sandbox evasion + suspicious behavior", "severity": "HIGH", - "evidence": "Anti: L2788: original_trace = sys.gettrace() | L2790: sys.settrace(None) | L2793: sys.settrace(original_trace)\nSubprocess: L1486: output = subprocess.run(cmd, capture_output=True, text=True) | L2889: res = subprocess.run(cmd, cwd=cwd, capture_output=True, text=True,\nL2890: errors=\"replace\", **kwargs)\nExec: L1352: exec(astr, dict) | L1640: exec(code, globs, locs)", - "evidence_hash": "9c6961817e5b1751e572dfe0858286703bb835870ecdfd6a7a9fdd8372a5dd2b" + "evidence": "Anti: L2777: original_trace = sys.gettrace() | L2779: sys.settrace(None) | L2782: sys.settrace(original_trace)\nSubprocess: L1478: output = subprocess.run(cmd, capture_output=True, text=True)\nExec: L1346: exec(astr, dict) | L1632: exec(code, globs, locs)", + "evidence_hash": "27468a6828101c6c026ae25aca8aa90ef485fd62b2c8f0967479edae9c965844" }, { "package": "numpy", @@ -1199,7 +1271,7 @@ "file": "PIL/Image.py", "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", "severity": "HIGH", - "evidence": "Obfusc: L422: __import__(f\"{__spec__.parent}.{plugin}\", globals(), locals(), []) | L490: __import__(f\"{__spec__.parent}.{plugin}\", globals(), locals(), [])\nExec: L3772: def eval(image: Image, *args: Callable[[int], float]) -> Image:", + "evidence": "Obfusc: L422: __import__(f\"{__spec__.parent}.{plugin}\", globals(), locals(), []) | L490: __import__(f\"{__spec__.parent}.{plugin}\", globals(), locals(), [])\nExec: L3776: def eval(image: Image, *args: Callable[[int], float]) -> Image:", "evidence_hash": "c2c1e7ae44e15862caf8de549d09db7b35e93282450f07ef61aaf5450a408c13" }, { @@ -1255,7 +1327,7 @@ "file": "setuptools/_distutils/compilers/C/base.py", "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", "severity": "HIGH", - "evidence": "Obfusc: L1286: __import__(module_name)\nExec: L1113: if lib_type not in eval(expected):", + "evidence": "Obfusc: L1287: __import__(module_name)\nExec: L1114: if lib_type not in eval(expected):", "evidence_hash": "368651e9818ed2d1bb009027d3bcfbf94ae30639c0882a6c2bddde97b8c4f1e5" }, { @@ -1271,7 +1343,7 @@ "file": "setuptools/tests/config/test_pyprojecttoml.py", "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", "severity": "HIGH", - "evidence": "Obfusc: L364: \"setup.py\": \"__import__('setuptools').setup(include_package_data=False)\",\nExec: L98: \"__main__.py\": \"def exec(): print('hello')\",", + "evidence": "Obfusc: L387: \"setup.py\": \"__import__('setuptools').setup(include_package_data=False)\",\nExec: L98: \"__main__.py\": \"def exec(): print('hello')\",", "evidence_hash": "067d41014f72a61d8b4adf25f3659d1f66a0e909f732223f48837aa7684df4e6" }, { @@ -1279,7 +1351,7 @@ "file": "setuptools/tests/test_editable_install.py", "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", "severity": "HIGH", - "evidence": "Obfusc: L120: SETUP_SCRIPT_STUB = \"__import__('setuptools').setup()\"\nExec: L449: exec(finder, loc, loc)", + "evidence": "Obfusc: L120: SETUP_SCRIPT_STUB = \"__import__('setuptools').setup()\"\nExec: L447: exec(finder, loc, loc)", "evidence_hash": "a78d7f5af7eb4ba92656cda258c195b92f6337c585c97d0823e47a9d4a2eb15d" }, { @@ -1322,12 +1394,20 @@ "evidence": "Obfusc: L919: c = compile(funcstr, filename, 'exec')\nExec: L163: module = eval(import_command) | L170: exec(import_command, {}, namespace) | L903: exec(ln, {}, namespace) | L909: exec(ln, {}, namespace) | L920: exec(c, namespace, funclocals)", "evidence_hash": "ab4f5819576a70038301668b8f3e4a781c4b757b146117d5d93eab1896a5a6cd" }, + { + "package": "tensorboard", + "file": "tensorboard/plugins/projector/tf_projector_plugin/projector_binary.js", + "check": "Python wheel ships large JS bundle (uncommon; manually review)", + "severity": "HIGH", + "evidence": "sha256: 53c38430766be25dc672a30846ac3b9eba86aee35eb0746785ec012647c7d9a2", + "evidence_hash": "2c6384e8115a6d5dacf1f84d8f724832d8dc59feb442bb98ffae0857c0ccb381" + }, { "package": "torch", "file": "torch/_dynamo/bytecode_debugger.py", "check": "Anti-analysis/sandbox evasion + suspicious behavior", "severity": "HIGH", - "evidence": "Anti: L1048: self._old_trace = sys.gettrace() | L1049: sys.settrace(self._settrace_callback) | L1106: sys.settrace(self._old_trace)\nExec: L683: result = eval(arg, frame_globals, eval_locals) | L708: result = eval(cmd, frame_globals, eval_locals) | L716: exec(cmd, frame_globals, eval_locals)", + "evidence": "Anti: L1052: self._old_trace = sys.gettrace() | L1053: sys.settrace(self._settrace_callback) | L1113: sys.settrace(self._old_trace)\nExec: L684: result = eval(arg, frame_globals, eval_locals) | L709: result = eval(cmd, frame_globals, eval_locals) | L717: exec(cmd, frame_globals, eval_locals)", "evidence_hash": "dc2afd1769d357c15b69802bd2799fafa059c0b1dcdd4937528fb5b601962f1b" }, { @@ -1343,7 +1423,7 @@ "file": "torch/fx/experimental/rewriter.py", "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", "severity": "HIGH", - "evidence": "Obfusc: L46: code = compile(dest_ast, \"\", \"exec\")\nExec: L49: exec(code, globals_dict)", + "evidence": "Obfusc: L44: code = compile(dest_ast, \"\", \"exec\")\nExec: L47: exec(code, globals_dict)", "evidence_hash": "76374f96feed416eec390458843621f33524cfb8d93ef0f3eb4cb1b47d0ad748" }, { @@ -1359,7 +1439,7 @@ "file": "torch/package/package_importer.py", "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", "severity": "HIGH", - "evidence": "Obfusc: L602: def __import__(self, name, globals=None, locals=None, fromlist=(), level=0):\nExec: L412: exec(code, ns)", + "evidence": "Obfusc: L599: def __import__(self, name, globals=None, locals=None, fromlist=(), level=0):\nExec: L412: exec(code, ns)", "evidence_hash": "c7c0650f0c74a086d224112f77ee76634b8f47afc047ce27fee8c7fc45560512" }, { @@ -1391,7 +1471,7 @@ "file": "tests/test_mlx_trainer_internals.py", "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", "severity": "HIGH", - "evidence": "Obfusc: L430: assert ppl == pytest.approx(__import__(\"math\").exp(2.5))\nExec: L408: def eval(self):", + "evidence": "Obfusc: L1158: assert ppl == pytest.approx(__import__(\"math\").exp(2.5))\nExec: L1136: def eval(self):", "evidence_hash": "c409327ef6420cc0c7224506fcb82b11bbc9838a6f2f97c9c2cfc00a40c4cdbf" }, { @@ -1407,7 +1487,7 @@ "file": "unsloth_zoo/compiler.py", "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", "severity": "HIGH", - "evidence": "Obfusc: L1013: _mod = __import__(model_location, fromlist=items) | L4291: f\" {chr(92)}{chr(92)} /| Num examples = {num_examples:,} | Num Epochs = {num_train_epochs:,} | Total steps = {max_steps:,}\\\\n\"\\\\\nL4292: f\"O^O/ {chr(92)}_/ {chr(92)} Batch size per device = {self._train_batch_size:,} | Gradient accumulation steps = {args.gradient_accumulation_steps}\\\\n\"\\\\\nL4293: f\"{chr(92)} / Data Parallel GPUs = {args.world_size} | Total batch size ({self._train_batch_size} x {args.gradient_accumulation_steps} x {args.world_size}) = {total_train_batch_size: sha256:b7ea9cdbe360ad323911014caf12a9d4ec87cb36195a511080e4e484c2fb80d2\nL4294: f' \"-____-\" Trainable parameters = {get_model_param_count(model, trainable_only=True):,} of {get_model_param_count(model):,} ({get_model_param_count(model, trainable_only=True)/get_model_p sha256:9e832c1e7b2815aa44dc42638d821cb9df22550db88d85bd4bfb29c13f984b53 | L4292: f\"O^O/ {chr(92)}_/ {chr(92)} Batch size per device = {self._train_batch_size:,} | Gradient accumulation steps = {args.gradient_accumulation_steps}\\\\n\"\\\\\nL4293: f\"{chr(92)} / Data Parallel GPUs = {args.world_size} | Total batch size ({self._train_batch_size} x {args.gradient_accumulation_steps} x {args.world_size}) = {total_train_batch_size: sha256:b7ea9cdbe360ad323911014caf12a9d4ec87cb36195a511080e4e484c2fb80d2\nL4294: f' \"-____-\" Trainable parameters = {get_model_param_count(model, trainable_only=True):,} of {get_model_param_count(model):,} ({get_model_param_count(model, trainable_only=True)/get_model_p sha256:9e832c1e7b2815aa44dc42638d821cb9df22550db88d85bd4bfb29c13f984b53 | L4291: f\" {chr(92)}{chr(92)} /| Num examples = {num_examples:,} | Num Epochs = {num_train_epochs:,} | Total steps = {max_steps:,}\\\\n\"\\\\\nL4292: f\"O^O/ {chr(92)}_/ {chr(92)} Batch size per device = {self._train_batch_size:,} | Gradient accumulation steps = {args.gradient_accumulation_steps}\\\\n\"\\\\\nL4293: f\"{chr(92)} / Data Parallel GPUs = {args.world_size} | Total batch size ({self._train_batch_size} x {args.gradient_accumulation_steps} x {args.world_size}) = {total_train_batch_size: sha256:b7ea9cdbe360ad323911014caf12a9d4ec87cb36195a511080e4e484c2fb80d2\nExec: L612: if eval(_dtype) is not None: | L613: dtype = eval(_dtype) | L955: _modeling_file = eval(model_location) | L1255: f = eval(f\"{model_location}.{module}\") | L1563: exec(f\"def raise_{j}(*args, **kwargs): print('{function}')\", globals(), locals()) | L1564: try: exec(f\"EMPTY_LOGITS.{function} = raise_{j}\", globals(), locals()) | L2699: exec(f\"import {parent}\", locals(), globals()) | L2830: dir(eval(parent)), | L2834: exec(f\"{parent}.{child}.forward = forward\", globals(), locals()) | L2908: module = eval(f\"modeling_file.{module}\") | L2935: inner_class = eval(f\"modeling_file.{inner_class}\") | L3065: exec(f\"from timm.layers.norm_act import {norm}\") | L3073: forward = eval(norm).forward | L3079: exec(f\"timm.layers.norm_act.{norm}.forward = forward\") | L3096: exec(f\"from timm.models._efficientnet_blocks import {block}\") | L3104: forward = eval(block).forward | L3110: exec(f\"timm.models._efficientnet_blocks.{block}.forward = forward\") | L3385: exec(f\"import {model_location}\", globals()) | L3388: modeling_file = eval(model_location) | L3401: exec(\nL3402: \"model_logger.addFilter(HideLoggingMessage('`use_cache`'))\", globals(), locals()\nL3403: ) | L3405: exec(\nL3406: \"model_logger.addFilter(HideLoggingMessage('compile_config'))\",\nL3407: globals(),\nL3408: locals(),\nL3409: ) | L3560: source = eval(f\"modeling_file.{module}\") | L3574: source = eval(f\"modeling_file.{module}\") | L3675: source = eval(f\"modeling_file.{module}\") | L3713: source = eval(f\"{model_location}.{module}\") | L3784: source = eval(f\"{model_location}.{module}\") | L3832: source = eval(f\"{model_location}.{module}\") | L4054: source = eval(f\"{model_location}.{module}\") | L4065: exec(\nL4066: f\"{model_location}.{module}._update_causal_mask = no_update_causal_mask\",\nL4067: globals(),\nL4068: ) | L4131: source = eval(f\"{model_location}.{module}\") | L4172: module_cls = eval(f\"{model_location}.{module}\") | L4209: module_cls = eval(f\"{model_location}.{module}\") | L4276: exec(\nL4277: \"from transformers.trainer import (\" + \", \".join(x for x in good_items) + \")\",\nL4278: globals(),\nL4279: ) | L4341: exec(inner_training_loop, globals()) | L4349: function = eval(f\"{model_location}.{module}\") | L4427: function = eval(f\"{model_location}.{module}\") | L4562: source = eval(f\"{model_location}.torch\") | L4569: function = eval(f\"source.nn.{module}\") | L4628: exec(\nL4629: f\"{model_location}.torch.nn.{module}.forward = forward\",\nL4630: globals(),\nL4631: locals(),\nL4632: ) | L4634: exec(\nL4635: f\"{model_location}.nn.{module}.forward = forward\",\nL4636: globals(),\nL4637: locals(),\nL4638: ) | L4642: exec(\nL4643: f\"combined_module.torch.nn.{module}.forward = forward\",\nL4644: globals(),\nL4645: locals(),\nL4646: ) | L4648: exec(\nL4649: f\"combined_module.nn.{module}.forward = forward\",\nL4650: globals(),\nL4651: locals(),\nL4652: ) | L4669: exec(\nL4670: f\"{model_location}.{module} = combined_module.{module}\",\nL4671: globals(),\nL4672: locals(),\nL4673: ) | L4683: check_dicts = dir(eval(f\"{model_location}\")) | L4685: item = eval(f\"{model_location}.{check}\") | L4695: exec(\nL4696: f\"{model_location}.{check}['{key}'] = combined_module.{replaced_class}\",\nL4697: globals(),\nL4698: locals(),\nL4699: )", + "evidence": "Obfusc: L1013: _mod = __import__(model_location, fromlist=items) | L4295: f\" {chr(92)}{chr(92)} /| Num examples = {num_examples:,} | Num Epochs = {num_train_epochs:,} | Total steps = {max_steps:,}\\\\n\"\\\\\nL4296: f\"O^O/ {chr(92)}_/ {chr(92)} Batch size per device = {self._train_batch_size:,} | Gradient accumulation steps = {args.gradient_accumulation_steps}\\\\n\"\\\\\nL4297: f\"{chr(92)} / Data Parallel GPUs = {args.world_size} | Total batch size ({self._train_batch_size} x {args.gradient_accumulation_steps} x {args.world_size}) = {total_train_batch_size: sha256:b7ea9cdbe360ad323911014caf12a9d4ec87cb36195a511080e4e484c2fb80d2\nL4298: f' \"-____-\" Trainable parameters = {get_model_param_count(model, trainable_only=True):,} of {get_model_param_count(model):,} ({get_model_param_count(model, trainable_only=True)/get_model_p sha256:9e832c1e7b2815aa44dc42638d821cb9df22550db88d85bd4bfb29c13f984b53 | L4296: f\"O^O/ {chr(92)}_/ {chr(92)} Batch size per device = {self._train_batch_size:,} | Gradient accumulation steps = {args.gradient_accumulation_steps}\\\\n\"\\\\\nL4297: f\"{chr(92)} / Data Parallel GPUs = {args.world_size} | Total batch size ({self._train_batch_size} x {args.gradient_accumulation_steps} x {args.world_size}) = {total_train_batch_size: sha256:b7ea9cdbe360ad323911014caf12a9d4ec87cb36195a511080e4e484c2fb80d2\nL4298: f' \"-____-\" Trainable parameters = {get_model_param_count(model, trainable_only=True):,} of {get_model_param_count(model):,} ({get_model_param_count(model, trainable_only=True)/get_model_p sha256:9e832c1e7b2815aa44dc42638d821cb9df22550db88d85bd4bfb29c13f984b53 | L4295: f\" {chr(92)}{chr(92)} /| Num examples = {num_examples:,} | Num Epochs = {num_train_epochs:,} | Total steps = {max_steps:,}\\\\n\"\\\\\nL4296: f\"O^O/ {chr(92)}_/ {chr(92)} Batch size per device = {self._train_batch_size:,} | Gradient accumulation steps = {args.gradient_accumulation_steps}\\\\n\"\\\\\nL4297: f\"{chr(92)} / Data Parallel GPUs = {args.world_size} | Total batch size ({self._train_batch_size} x {args.gradient_accumulation_steps} x {args.world_size}) = {total_train_batch_size: sha256:b7ea9cdbe360ad323911014caf12a9d4ec87cb36195a511080e4e484c2fb80d2\nExec: L612: if eval(_dtype) is not None: | L613: dtype = eval(_dtype) | L955: _modeling_file = eval(model_location) | L1255: f = eval(f\"{model_location}.{module}\") | L1563: exec(f\"def raise_{j}(*args, **kwargs): print('{function}')\", globals(), locals()) | L1564: try: exec(f\"EMPTY_LOGITS.{function} = raise_{j}\", globals(), locals()) | L2699: exec(f\"import {parent}\", locals(), globals()) | L2830: dir(eval(parent)), | L2834: exec(f\"{parent}.{child}.forward = forward\", globals(), locals()) | L2908: module = eval(f\"modeling_file.{module}\") | L2935: inner_class = eval(f\"modeling_file.{inner_class}\") | L3065: exec(f\"from timm.layers.norm_act import {norm}\") | L3073: forward = eval(norm).forward | L3079: exec(f\"timm.layers.norm_act.{norm}.forward = forward\") | L3096: exec(f\"from timm.models._efficientnet_blocks import {block}\") | L3104: forward = eval(block).forward | L3110: exec(f\"timm.models._efficientnet_blocks.{block}.forward = forward\") | L3389: exec(f\"import {model_location}\", globals()) | L3392: modeling_file = eval(model_location) | L3405: exec(\nL3406: \"model_logger.addFilter(HideLoggingMessage('`use_cache`'))\", globals(), locals()\nL3407: ) | L3409: exec(\nL3410: \"model_logger.addFilter(HideLoggingMessage('compile_config'))\",\nL3411: globals(),\nL3412: locals(),\nL3413: ) | L3564: source = eval(f\"modeling_file.{module}\") | L3578: source = eval(f\"modeling_file.{module}\") | L3679: source = eval(f\"modeling_file.{module}\") | L3717: source = eval(f\"{model_location}.{module}\") | L3788: source = eval(f\"{model_location}.{module}\") | L3836: source = eval(f\"{model_location}.{module}\") | L4058: source = eval(f\"{model_location}.{module}\") | L4069: exec(\nL4070: f\"{model_location}.{module}._update_causal_mask = no_update_causal_mask\",\nL4071: globals(),\nL4072: ) | L4135: source = eval(f\"{model_location}.{module}\") | L4176: module_cls = eval(f\"{model_location}.{module}\") | L4213: module_cls = eval(f\"{model_location}.{module}\") | L4280: exec(\nL4281: \"from transformers.trainer import (\" + \", \".join(x for x in good_items) + \")\",\nL4282: globals(),\nL4283: ) | L4345: exec(inner_training_loop, globals()) | L4353: function = eval(f\"{model_location}.{module}\") | L4431: function = eval(f\"{model_location}.{module}\") | L4566: source = eval(f\"{model_location}.torch\") | L4573: function = eval(f\"source.nn.{module}\") | L4632: exec(\nL4633: f\"{model_location}.torch.nn.{module}.forward = forward\",\nL4634: globals(),\nL4635: locals(),\nL4636: ) | L4638: exec(\nL4639: f\"{model_location}.nn.{module}.forward = forward\",\nL4640: globals(),\nL4641: locals(),\nL4642: ) | L4646: exec(\nL4647: f\"combined_module.torch.nn.{module}.forward = forward\",\nL4648: globals(),\nL4649: locals(),\nL4650: ) | L4652: exec(\nL4653: f\"combined_module.nn.{module}.forward = forward\",\nL4654: globals(),\nL4655: locals(),\nL4656: ) | L4673: exec(\nL4674: f\"{model_location}.{module} = combined_module.{module}\",\nL4675: globals(),\nL4676: locals(),\nL4677: ) | L4687: check_dicts = dir(eval(f\"{model_location}\")) | L4689: item = eval(f\"{model_location}.{check}\") | L4699: exec(\nL4700: f\"{model_location}.{check}['{key}'] = combined_module.{replaced_class}\",\nL4701: globals(),\nL4702: locals(),\nL4703: )", "evidence_hash": "ec1875fd32d00fe885e566ebda75163e46e838ca31020abb57e0991892c2bdf7" }, { @@ -1423,8 +1503,8 @@ "file": "unsloth_zoo/mlx/loader.py", "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", "severity": "HIGH", - "evidence": "Obfusc: L2218: _mod = __import__(module_name, fromlist=[\"_\"])\nExec: L140: mx.eval(model.parameters()) | L176: mx.eval(model.parameters()) | L2022: model.eval() | L2605: mx.eval(model.parameters()) | L2721: mx.eval(module.weight) | L4030: mx.eval(model.parameters()) | L4058: mx.eval(model.parameters()) | L4178: mx.eval(model.parameters())", - "evidence_hash": "9b29dade82912216c8b4808aa293b79749aa80ef1d2be35edd93bec7632810f1" + "evidence": "Obfusc: L2869: _mod = __import__(module_name, fromlist=[\"_\"])\nExec: L148: mx.eval(model.parameters()) | L180: mx.eval(model.parameters()) | L732: mx.eval(model.parameters()) | L733: mx.eval(mx.distributed.all_sum(mx.array(1.0), stream=mx.cpu)) | L799: mx.eval(model.parameters()) | L802: mx.eval(mx.distributed.all_sum(mx.array(1.0), stream=mx.cpu)) | L2673: model.eval() | L3256: mx.eval(model.parameters()) | L3372: mx.eval(module.weight) | L5666: mx.eval(model.parameters()) | L5716: mx.eval(model.parameters()) | L5859: mx.eval(model.parameters())", + "evidence_hash": "7b44760032c5df6d379ccfdd0bff3d23f857f64e08210fa0fba8d2881d457634" }, { "package": "unsloth-zoo", @@ -1439,7 +1519,7 @@ "file": "unsloth_zoo/saving_utils.py", "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", "severity": "HIGH", - "evidence": "Obfusc: L3241: module = __import__('transformers', fromlist=[model_class_name])\nExec: L3123: exec(f\"from transformers.modeling_utils import ({', '.join(functions)})\", locals(), globals()) | L3169: exec(save_pretrained, globals(), functions)", + "evidence": "Obfusc: L4015: module = __import__('transformers', fromlist=[model_class_name])\nExec: L3897: exec(f\"from transformers.modeling_utils import ({', '.join(functions)})\", locals(), globals()) | L3943: exec(save_pretrained, globals(), functions)", "evidence_hash": "530b2383acd9fe8330aa65cd0bf86164aaacd47770e7c8d0752195bee36396ec" }, { @@ -1449,118 +1529,6 @@ "severity": "HIGH", "evidence": "Obfusc: L836: code = compile(module, \"\", \"exec\")\nExec: L736: exec(code, globs, locs)", "evidence_hash": "5c0992c90f05c772abd94d00784f157de337e1f8567f8b3aee1b15e46c96cd5d" - }, - { - "package": "multiprocess", - "file": "multiprocess/tests/__init__.py", - "check": "Reverse shell / bind shell pattern", - "severity": "CRITICAL", - "evidence": "L3355: os.dup2(conn.fileno(), i) | L3387: \"test needs os.dup2()\") | L3405: os.dup2(fd, newfd) | L19: import socket sha256:26a745abdc7e89da28ab943394234d8ccb415e805477c3cc1f7d4766341a4c4c", - "evidence_hash": "a6b9bb85e9bb6682ab0dea4f95fd9266e8802f118c76d86dd87f7ab5864872cf" - }, - { - "package": "unsloth-zoo", - "file": "scripts/scan_packages.py", - "check": "Writes to /tmp and executes (staged dropper)", - "severity": "CRITICAL", - "evidence": "L308: r\"/tmp/\\S+.*(?:subprocess|os\\.system|os\\.popen|Popen|chmod.*\\+x)\", | L308: r\"/tmp/\\S+.*(?:subprocess|os\\.system|os\\.popen|Popen|chmod.*\\+x)\", sha256:78268349021e21bedcd2eaaa5b4a71b0de1d52e023ada914dfdc09515ee1aad8", - "evidence_hash": "590fe1c96c442fbea5eb8642650257bc0b0199e919b9bacdb11dfa767b6fe839" - }, - { - "package": "multiprocess", - "file": "multiprocess/tests/__init__.py", - "check": "Reverse shell / bind shell pattern", - "severity": "CRITICAL", - "evidence": "L3521: os.dup2(conn.fileno(), i) | L3553: \"test needs os.dup2()\") | L3571: os.dup2(fd, newfd) | L20: import socket sha256:07d2933301c0dbeeb6e42381687827d8dd7cfd7471986c559ca64283d5ae6e24", - "evidence_hash": "db1f4ca69865ec3911d7450fe11d212b817139deda21cd7a4ee32d547a8dc452" - }, - { - "package": "fastapi", - "file": "fastapi/routing.py", - "check": "C2 polling/beaconing loop detected", - "severity": "CRITICAL", - "evidence": "L586: while True: sha256:bef9ea429314fad39e063895a37dc5cfe9b04561f3d1acbb3c99abb4e92e6cfe", - "evidence_hash": "b15773e1bc249713156a349278ea60f7c0e3dd7d537affe929ab51089e1942bb" - }, - { - "package": "tensorboard", - "file": "tensorboard/plugins/projector/tf_projector_plugin/projector_binary.js", - "check": "Python wheel ships large JS bundle (uncommon; manually review)", - "severity": "HIGH", - "evidence": "sha256: 53c38430766be25dc672a30846ac3b9eba86aee35eb0746785ec012647c7d9a2", - "evidence_hash": "2c6384e8115a6d5dacf1f84d8f724832d8dc59feb442bb98ffae0857c0ccb381" - }, - { - "package": "fastapi", - "file": "fastapi/routing.py", - "check": "C2 polling/beaconing loop detected", - "severity": "CRITICAL", - "evidence": "L586: while True: sha256:251135b5ebfdd1248916449f32262575e003ef64382501c65b7e4061d67bda45", - "evidence_hash": "365aef4449c8089753d9398417cd76ab762cef547d75db70d87bca9c0b550ab5" - }, - { - "package": "fastmcp-slim", - "file": "fastmcp/cli/apps_dev.py", - "check": "Enumerates filesystem AND makes network calls", - "severity": "CRITICAL", - "evidence": "FS: L637: history.replaceState(null, \"\", url); sha256:17068ba5bfed62c3a3007ec8bf3e0ea41ef6529b9e6112064d9afb3be9231436\nNetwork: L1304: with httpx.Client(timeout=30.0) as client: | L1318: with httpx.Client(timeout=30.0) as client: | L1348: with httpx.Client(timeout=30.0) as client: | L1549: client = httpx.AsyncClient(\nL1550: timeout=httpx.Timeout(60.0, read=None), trust_env=False\nL1551: ) | L1713: async with httpx.AsyncClient(trust_env=False) as client: | L1781: with socket.socket(family, socket.SOCK_STREAM) as s:", - "evidence_hash": "e5325edfada6499540e6f0c24a0868979d275522e2b6a180aa9b5dd3280681b4" - }, - { - "package": "huggingface-hub", - "file": "huggingface_hub/_sandbox.py", - "check": "C2 polling/beaconing loop detected", - "severity": "CRITICAL", - "evidence": "L1179: while True: sha256:33ceddf9e42aae207e891e97808c518e92a0b27ab60e4326256717bfb25a3a38", - "evidence_hash": "802fd41d8bb17bf425e99d128c0351c820103a5efb74690a4086e542a71437b8" - }, - { - "package": "huggingface-hub", - "file": "huggingface_hub/_sandbox.py", - "check": "Writes to /tmp and executes (staged dropper)", - "severity": "CRITICAL", - "evidence": "L83: d=/tmp/.sbx-server\nL84: if command -v wget >/dev/null 2>&1; then wget -q --header \"Authorization: Bearer $SBX_DL_TOKEN\" -O \"$d\" \"$SBX_SERVER_URL\"\nL85: elif command -v curl >/dev/null 2>&1; then curl -fsSL -H \"Authorization: Bearer $SBX_DL_TOKEN\" -o \"$d\" \"$SBX_SERVER_URL\"\nL86: else cp \"$SBX_SERVER_MOUNT/sbx-server\" \"$d\"; fi\nL87: chmod +x \"$d\"", - "evidence_hash": "6908a3fe328fa94ee22a119998d6ad07cfa1ba4efa2628acf240f4204fd76e22" - }, - { - "package": "huggingface-hub", - "file": "huggingface_hub/hf_api.py", - "check": "C2 polling/beaconing loop detected", - "severity": "CRITICAL", - "evidence": "L4613: while True: sha256:f764b6ca3118b23c7c0e670e77178c022a6905f825d7df6e528545fa10aae8f6", - "evidence_hash": "9c85d50c227285fa8dc69512999cbb082258cda4b299c7d0e0f69f5aff7accd4" - }, - { - "package": "huggingface-hub", - "file": "huggingface_hub/utils/_http.py", - "check": "C2 polling/beaconing loop detected", - "severity": "CRITICAL", - "evidence": "L462: while True: sha256:c75d1ee228cf7703a8c28551d649395a1f89f69a3aba69413f5bbcbd10c31958", - "evidence_hash": "d4d5f83fed39b87898cf776d5dad0bf1a6388a932f5fb7997d1070b50e46213e" - }, - { - "package": "cffi", - "file": "cffi/_cffi_gen_src.py", - "check": "Advanced obfuscation (marshal/compile/zlib) + exec/eval", - "severity": "HIGH", - "evidence": "Obfusc: L52: compiled = compile(source=pysrc, filename=filename, mode='exec')\nExec: L53: exec(compiled, globs, globs)", - "evidence_hash": "c429e4c977a61db6b7c717b5a552fce74eda622213e49eb5467a3782fd746fb9" - }, - { - "package": "multiprocess", - "file": "multiprocess/forkserver.py", - "check": "Reverse shell / bind shell pattern", - "severity": "CRITICAL", - "evidence": "L6: import socket sha256:6c707119169286c9a798e2c8d13a48614e481d8a503950916fd4ffb4c94d3182", - "evidence_hash": "50fec0f0522a8e4e636bf348b752002d7935d8455af31fb78c6f11e2eba19f6d" - }, - { - "package": "multiprocess", - "file": "multiprocess/tests/__init__.py", - "check": "Reverse shell / bind shell pattern", - "severity": "CRITICAL", - "evidence": "L3569: os.dup2(conn.fileno(), i) | L3601: \"test needs os.dup2()\") | L3619: os.dup2(fd, newfd) | L20: import socket sha256:c824dc0f409f242420c3fbb324790c53cb3078d2c8b07ee8f2a05694b01c2946", - "evidence_hash": "3878a2b430c175dbc5877a95195bfe52f9588ff73fb74e2261ed5e33087915ad" } ] } From fb5dc91bb4f33f8a4c5a41c89cbe88c93394e970 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 9 Jul 2026 05:09:16 -0700 Subject: [PATCH 022/372] Studio: remove dead direct_linux_release_plan path (#7030) parse_direct_linux_release_bundle and direct_linux_release_plan are no longer reached by any live code path. Fork Linux installs resolve through _fork_manifest_release_plans -> _linux_published_attempts, and the upstream (ggml-org) path uses direct_upstream_release_plan. The dead parser also called _resolve_linux_bundle_profile, which no longer exists, so its CUDA branch would raise NameError if ever executed. Drop both functions and the obsolete TestDirectLinuxNvidiaCpuGate; its live equivalent TestLinuxPublishedAttemptsNvidiaCpuGate already covers the NVIDIA no-silent-CPU behaviour. --- studio/install_llama_prebuilt.py | 156 ------------------- tests/studio/install/test_selection_logic.py | 62 -------- 2 files changed, 218 deletions(-) diff --git a/studio/install_llama_prebuilt.py b/studio/install_llama_prebuilt.py index 856ba71478..40caebc040 100644 --- a/studio/install_llama_prebuilt.py +++ b/studio/install_llama_prebuilt.py @@ -1286,162 +1286,6 @@ def synthetic_checksums_for_release( ) -def parse_direct_linux_release_bundle( - repo: str, release: dict[str, Any] -) -> PublishedReleaseBundle | None: - release_tag = release.get("tag_name") - if not isinstance(release_tag, str) or not release_tag: - return None - - assets = release_asset_map(release) - artifacts: list[PublishedLlamaArtifact] = [] - inferred_labels: list[str] = [] - - linux_asset_re = re.compile( - r"^app-(?P