From 1d8160376e169d13c386b7ef4bc1fdc8f855de68 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 6 Apr 2026 09:46:35 -0700 Subject: [PATCH 01/60] Bump minimum unsloth version to 2026.4.4 in install scripts (#4876) --- install.ps1 | 10 +++++----- install.sh | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/install.ps1 b/install.ps1 index 5ddb42ea7e..a2acd6c4ea 100644 --- a/install.ps1 +++ b/install.ps1 @@ -819,7 +819,7 @@ shell.Run cmd, 0, False if ($SkipTorch) { # No-torch: install unsloth + unsloth-zoo with --no-deps, then # runtime deps (typer, safetensors, transformers, etc.) with --no-deps. - $baseInstallExit = Invoke-InstallCommand { uv pip install --python $VenvPython --no-deps --reinstall-package unsloth --reinstall-package unsloth-zoo "unsloth>=2026.4.2" unsloth-zoo } + $baseInstallExit = Invoke-InstallCommand { uv pip install --python $VenvPython --no-deps --reinstall-package unsloth --reinstall-package unsloth-zoo "unsloth>=2026.4.4" unsloth-zoo } if ($baseInstallExit -eq 0) { $NoTorchReq = Find-NoTorchRuntimeFile if ($NoTorchReq) { @@ -827,7 +827,7 @@ shell.Run cmd, 0, False } } } else { - $baseInstallExit = Invoke-InstallCommand { uv pip install --python $VenvPython --reinstall-package unsloth --reinstall-package unsloth-zoo "unsloth>=2026.4.2" unsloth-zoo } + $baseInstallExit = Invoke-InstallCommand { uv pip install --python $VenvPython --reinstall-package unsloth --reinstall-package unsloth-zoo "unsloth>=2026.4.4" unsloth-zoo } } if ($baseInstallExit -ne 0) { Write-Host "[ERROR] Failed to install unsloth (exit code $baseInstallExit)" -ForegroundColor Red @@ -857,7 +857,7 @@ shell.Run cmd, 0, False if ($SkipTorch) { # No-torch: install unsloth + unsloth-zoo with --no-deps, then # runtime deps (typer, safetensors, transformers, etc.) with --no-deps. - $baseInstallExit = Invoke-InstallCommand { uv pip install --python $VenvPython --no-deps --upgrade-package unsloth --upgrade-package unsloth-zoo "unsloth>=2026.4.2" unsloth-zoo } + $baseInstallExit = Invoke-InstallCommand { uv pip install --python $VenvPython --no-deps --upgrade-package unsloth --upgrade-package unsloth-zoo "unsloth>=2026.4.4" unsloth-zoo } if ($baseInstallExit -eq 0) { $NoTorchReq = Find-NoTorchRuntimeFile if ($NoTorchReq) { @@ -865,7 +865,7 @@ shell.Run cmd, 0, False } } } elseif ($StudioLocalInstall) { - $baseInstallExit = Invoke-InstallCommand { uv pip install --python $VenvPython --upgrade-package unsloth "unsloth>=2026.4.2" unsloth-zoo } + $baseInstallExit = Invoke-InstallCommand { uv pip install --python $VenvPython --upgrade-package unsloth "unsloth>=2026.4.4" unsloth-zoo } } else { $baseInstallExit = Invoke-InstallCommand { uv pip install --python $VenvPython --upgrade-package unsloth "$PackageName" } } @@ -886,7 +886,7 @@ shell.Run cmd, 0, False # Fallback: GPU detection failed to produce a URL -- let uv resolve torch substep "installing unsloth (this may take a few minutes)..." if ($StudioLocalInstall) { - $baseInstallExit = Invoke-InstallCommand { uv pip install --python $VenvPython unsloth-zoo "unsloth>=2026.4.2" --torch-backend=auto } + $baseInstallExit = Invoke-InstallCommand { uv pip install --python $VenvPython unsloth-zoo "unsloth>=2026.4.4" --torch-backend=auto } if ($baseInstallExit -ne 0) { Write-Host "[ERROR] Failed to install unsloth (exit code $baseInstallExit)" -ForegroundColor Red return diff --git a/install.sh b/install.sh index 053f334d2b..ea53ecc6d6 100755 --- a/install.sh +++ b/install.sh @@ -1040,7 +1040,7 @@ if [ "$_MIGRATED" = true ]; then # to prevent transitive torch resolution. run_install_cmd "install unsloth (migrated no-torch)" uv pip install --python "$_VENV_PY" --no-deps \ --reinstall-package unsloth --reinstall-package unsloth-zoo \ - "unsloth>=2026.4.2" unsloth-zoo + "unsloth>=2026.4.4" unsloth-zoo _NO_TORCH_RT="$(_find_no_torch_runtime)" if [ -n "$_NO_TORCH_RT" ]; then run_install_cmd "install no-torch runtime deps" uv pip install --python "$_VENV_PY" --no-deps -r "$_NO_TORCH_RT" @@ -1048,7 +1048,7 @@ if [ "$_MIGRATED" = true ]; then else run_install_cmd "install unsloth (migrated)" uv pip install --python "$_VENV_PY" \ --reinstall-package unsloth --reinstall-package unsloth-zoo \ - "unsloth>=2026.4.2" unsloth-zoo + "unsloth>=2026.4.4" unsloth-zoo fi if [ "$STUDIO_LOCAL_INSTALL" = true ]; then substep "overlaying local repo (editable)..." @@ -1070,7 +1070,7 @@ elif [ -n "$TORCH_INDEX_URL" ]; then # runtime deps (typer, safetensors, transformers, etc.) with --no-deps. run_install_cmd "install unsloth (no-torch)" uv pip install --python "$_VENV_PY" --no-deps \ --upgrade-package unsloth --upgrade-package unsloth-zoo \ - "unsloth>=2026.4.2" unsloth-zoo + "unsloth>=2026.4.4" unsloth-zoo _NO_TORCH_RT="$(_find_no_torch_runtime)" if [ -n "$_NO_TORCH_RT" ]; then run_install_cmd "install no-torch runtime deps" uv pip install --python "$_VENV_PY" --no-deps -r "$_NO_TORCH_RT" @@ -1081,7 +1081,7 @@ elif [ -n "$TORCH_INDEX_URL" ]; then fi elif [ "$STUDIO_LOCAL_INSTALL" = true ]; then run_install_cmd "install unsloth (local)" uv pip install --python "$_VENV_PY" \ - --upgrade-package unsloth "unsloth>=2026.4.2" unsloth-zoo + --upgrade-package unsloth "unsloth>=2026.4.4" unsloth-zoo substep "overlaying local repo (editable)..." run_install_cmd "overlay local repo" uv pip install --python "$_VENV_PY" -e "$_REPO_ROOT" --no-deps else @@ -1092,7 +1092,7 @@ else # Fallback: GPU detection failed to produce a URL -- let uv resolve torch substep "installing unsloth (this may take a few minutes)..." if [ "$STUDIO_LOCAL_INSTALL" = true ]; then - run_install_cmd "install unsloth (auto torch backend)" uv pip install --python "$_VENV_PY" unsloth-zoo "unsloth>=2026.4.2" --torch-backend=auto + run_install_cmd "install unsloth (auto torch backend)" uv pip install --python "$_VENV_PY" unsloth-zoo "unsloth>=2026.4.4" --torch-backend=auto substep "overlaying local repo (editable)..." run_install_cmd "overlay local repo" uv pip install --python "$_VENV_PY" -e "$_REPO_ROOT" --no-deps else From f801e59c29db7b4028297ef987af6bfdaa464500 Mon Sep 17 00:00:00 2001 From: Roland Tannous <115670425+rolandtannous@users.noreply.github.com> Date: Tue, 7 Apr 2026 20:05:01 +0400 Subject: [PATCH 02/60] split venv_t5 into tiered 5.3.0/5.5.0 and fix trust_remote_code (#4878) * split venv_t5 into venv_t5_530 and venv_t5_550 for tiered transformers 5.x support * fix bfloat16 crash on T4 for FORCE_FLOAT32 models and disable trust_remote_code auto-enable for native t5 models * revert FORCE_FLOAT32 dtype change * restrict trust_remote_code auto-enable to Nemotron models only * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * use config.json model_type for tier detection, add unsloth/nvidia namespace guard * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Revert "[pre-commit.ci] auto fixes from pre-commit.com hooks" This reverts commit fb43d468e25379f28dd2477e6c24dd60cf55c099. * Revert "use config.json model_type for tier detection, add unsloth/nvidia namespace guard" This reverts commit fc49ae24531780a658049e6c238f146960f466b6. * add unsloth/nvidia namespace guard to Nemotron trust_remote_code auto-enable * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * reorder tier checks: all substring matches before config.json fetches * extract shared activate_transformers_for_subprocess into transformers_version.py * narrow Nemotron trust_remote_code to nemotron_h/nemotron-3-nano, add to export worker * clean venv_t5 dirs before re-install in setup.sh, clarify version alias comment * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * run venv_t5 migration outside deps fast-path gate in both setup scripts --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- studio/backend/core/export/worker.py | 41 +-- studio/backend/core/inference/worker.py | 44 +-- studio/backend/core/training/worker.py | 51 +-- .../tests/test_transformers_version.py | 148 ++++++++ studio/backend/utils/models/model_config.py | 5 +- studio/backend/utils/transformers_version.py | 340 ++++++++++++++---- studio/setup.ps1 | 135 ++++--- studio/setup.sh | 53 ++- 8 files changed, 601 insertions(+), 216 deletions(-) diff --git a/studio/backend/core/export/worker.py b/studio/backend/core/export/worker.py index 6af6ff1193..3f3dc955fa 100644 --- a/studio/backend/core/export/worker.py +++ b/studio/backend/core/export/worker.py @@ -30,37 +30,15 @@ logger = get_logger(__name__) def _activate_transformers_version(model_name: str) -> None: - """Activate the correct transformers version BEFORE any ML imports. - - If the model needs transformers 5.x, prepend the pre-installed .venv_t5/ - directory to sys.path. Otherwise do nothing (default 4.57.x in .venv/). - """ + """Activate the correct transformers version BEFORE any ML imports.""" # Ensure backend is on path for utils imports backend_path = str(Path(__file__).resolve().parent.parent.parent) if backend_path not in sys.path: sys.path.insert(0, backend_path) - from utils.transformers_version import ( - needs_transformers_5, - _resolve_base_model, - _ensure_venv_t5_exists, - _VENV_T5_DIR, - ) + from utils.transformers_version import activate_transformers_for_subprocess - resolved = _resolve_base_model(model_name) - if needs_transformers_5(resolved): - if not _ensure_venv_t5_exists(): - raise RuntimeError( - f"Cannot activate transformers 5.x: .venv_t5 missing at {_VENV_T5_DIR}" - ) - if _VENV_T5_DIR not in sys.path: - sys.path.insert(0, _VENV_T5_DIR) - logger.info("Activated transformers 5.x from %s", _VENV_T5_DIR) - # Propagate to child subprocesses (e.g. GGUF converter) - _pp = os.environ.get("PYTHONPATH", "") - os.environ["PYTHONPATH"] = _VENV_T5_DIR + (os.pathsep + _pp if _pp else "") - else: - logger.info("Using default transformers (4.57.x) for %s", model_name) + activate_transformers_for_subprocess(model_name) def _send_response(resp_queue: Any, response: dict) -> None: @@ -78,6 +56,19 @@ def _handle_load(backend, cmd: dict, resp_queue: Any) -> None: load_in_4bit = cmd.get("load_in_4bit", True) trust_remote_code = cmd.get("trust_remote_code", False) + # Auto-enable trust_remote_code for NemotronH/Nano models. + if not trust_remote_code: + _NEMOTRON_TRUST_SUBSTRINGS = ("nemotron_h", "nemotron-h", "nemotron-3-nano") + _cp_lower = checkpoint_path.lower() + if any(sub in _cp_lower for sub in _NEMOTRON_TRUST_SUBSTRINGS) and ( + _cp_lower.startswith("unsloth/") or _cp_lower.startswith("nvidia/") + ): + trust_remote_code = True + logger.info( + "Auto-enabled trust_remote_code for Nemotron model: %s", + checkpoint_path, + ) + try: _send_response( resp_queue, diff --git a/studio/backend/core/inference/worker.py b/studio/backend/core/inference/worker.py index 7f7291a56d..fbcce276ba 100644 --- a/studio/backend/core/inference/worker.py +++ b/studio/backend/core/inference/worker.py @@ -34,37 +34,15 @@ from utils.hardware import apply_gpu_ids def _activate_transformers_version(model_name: str) -> None: - """Activate the correct transformers version BEFORE any ML imports. - - If the model needs transformers 5.x, prepend the pre-installed .venv_t5/ - directory to sys.path. Otherwise do nothing (default 4.57.x in .venv/). - """ + """Activate the correct transformers version BEFORE any ML imports.""" # Ensure backend is on path for utils imports backend_path = str(Path(__file__).resolve().parent.parent.parent) if backend_path not in sys.path: sys.path.insert(0, backend_path) - from utils.transformers_version import ( - needs_transformers_5, - _resolve_base_model, - _ensure_venv_t5_exists, - _VENV_T5_DIR, - ) + from utils.transformers_version import activate_transformers_for_subprocess - resolved = _resolve_base_model(model_name) - if needs_transformers_5(resolved): - if not _ensure_venv_t5_exists(): - raise RuntimeError( - f"Cannot activate transformers 5.x: .venv_t5 missing at {_VENV_T5_DIR}" - ) - if _VENV_T5_DIR not in sys.path: - sys.path.insert(0, _VENV_T5_DIR) - logger.info("Activated transformers 5.x from %s", _VENV_T5_DIR) - # Propagate to child subprocesses (e.g. GGUF converter) - _pp = os.environ.get("PYTHONPATH", "") - os.environ["PYTHONPATH"] = _VENV_T5_DIR + (os.pathsep + _pp if _pp else "") - else: - logger.info("Using default transformers (4.57.x) for %s", model_name) + activate_transformers_for_subprocess(model_name) def _decode_image(image_base64: str): @@ -309,19 +287,21 @@ def _handle_load(backend, config: dict, resp_queue: Any) -> None: except Exception as e: logger.warning("Could not read adapter_config.json: %s", e) - # Auto-enable trust_remote_code for unsloth/* transformers 5.x models - # (matches the training worker logic in core/training/worker.py) + # Auto-enable trust_remote_code for NemotronH/Nano models only. + # NemotronH has config parsing bugs requiring trust_remote_code=True. + # Other transformers 5.x models are native and do NOT need it. + # NOTE: Must NOT match Llama-Nemotron (standard Llama architecture). + _NEMOTRON_TRUST_SUBSTRINGS = ("nemotron_h", "nemotron-h", "nemotron-3-nano") trust_remote_code = config.get("trust_remote_code", False) if not trust_remote_code: - from utils.transformers_version import needs_transformers_5 - model_name = config["model_name"] - if needs_transformers_5(model_name) and model_name.lower().startswith( - "unsloth/" + _mn_lower = model_name.lower() + if any(sub in _mn_lower for sub in _NEMOTRON_TRUST_SUBSTRINGS) and ( + _mn_lower.startswith("unsloth/") or _mn_lower.startswith("nvidia/") ): trust_remote_code = True logger.info( - "Auto-enabled trust_remote_code for unsloth/* transformers 5.x model: %s", + "Auto-enabled trust_remote_code for Nemotron model: %s", model_name, ) diff --git a/studio/backend/core/training/worker.py b/studio/backend/core/training/worker.py index 0454eada89..a461972eca 100644 --- a/studio/backend/core/training/worker.py +++ b/studio/backend/core/training/worker.py @@ -306,37 +306,15 @@ def _ensure_mamba_ssm(event_queue: Any, model_name: str) -> None: def _activate_transformers_version(model_name: str) -> None: - """Activate the correct transformers version BEFORE any ML imports. - - If the model needs transformers 5.x, prepend the pre-installed .venv_t5/ - directory to sys.path. Otherwise do nothing (default 4.57.x in .venv/). - """ + """Activate the correct transformers version BEFORE any ML imports.""" # Ensure backend is on path for utils imports backend_path = str(Path(__file__).resolve().parent.parent.parent) if backend_path not in sys.path: sys.path.insert(0, backend_path) - from utils.transformers_version import ( - needs_transformers_5, - _resolve_base_model, - _ensure_venv_t5_exists, - _VENV_T5_DIR, - ) + from utils.transformers_version import activate_transformers_for_subprocess - resolved = _resolve_base_model(model_name) - if needs_transformers_5(resolved): - if not _ensure_venv_t5_exists(): - raise RuntimeError( - f"Cannot activate transformers 5.x: .venv_t5 missing at {_VENV_T5_DIR}" - ) - if _VENV_T5_DIR not in sys.path: - sys.path.insert(0, _VENV_T5_DIR) - logger.info("Activated transformers 5.x from %s", _VENV_T5_DIR) - # Propagate to child subprocesses (e.g. GGUF converter) - _pp = os.environ.get("PYTHONPATH", "") - os.environ["PYTHONPATH"] = _VENV_T5_DIR + (os.pathsep + _pp if _pp else "") - else: - logger.info("Using default transformers (4.57.x) for %s", model_name) + activate_transformers_for_subprocess(model_name) def run_training_process( @@ -386,25 +364,22 @@ def run_training_process( ) return - # ── 1a. Auto-enable trust_remote_code for unsloth/* transformers 5.x models ── - # Some newer architectures (e.g. NemotronH) have config parsing bugs in - # transformers that require trust_remote_code=True as a workaround. - # Only auto-enable for unsloth/* prefixed models (trusted source). - # Exclude Gemma 4 since it is a native transformers 5.5 model and - # trust_remote_code=True would bypass the compiler (disabling fused CE). - from utils.transformers_version import needs_transformers_5 - + # ── 1a. Auto-enable trust_remote_code for NemotronH/Nano models ── + # NemotronH has config parsing bugs in transformers that require + # trust_remote_code=True as a workaround. Other transformers 5.x models + # (Qwen3.5, Gemma 4, etc.) are native and do NOT need it — enabling it + # bypasses the compiler (disabling fused CE). + # NOTE: Must NOT match Llama-Nemotron (standard Llama architecture). + _NEMOTRON_TRUST_SUBSTRINGS = ("nemotron_h", "nemotron-h", "nemotron-3-nano") _lowered = model_name.lower() - _is_native_t5 = any(x in _lowered for x in ("gemma-4", "gemma4")) if ( - needs_transformers_5(model_name) - and _lowered.startswith("unsloth/") - and not _is_native_t5 + any(sub in _lowered for sub in _NEMOTRON_TRUST_SUBSTRINGS) + and (_lowered.startswith("unsloth/") or _lowered.startswith("nvidia/")) and not config.get("trust_remote_code", False) ): config["trust_remote_code"] = True logger.info( - "Auto-enabled trust_remote_code for unsloth/* transformers 5.x model: %s", + "Auto-enabled trust_remote_code for Nemotron model: %s", model_name, ) diff --git a/studio/backend/tests/test_transformers_version.py b/studio/backend/tests/test_transformers_version.py index f3dae537c7..c031c2fea3 100644 --- a/studio/backend/tests/test_transformers_version.py +++ b/studio/backend/tests/test_transformers_version.py @@ -31,8 +31,11 @@ sys.modules.setdefault("loggers", _loggers_stub) from utils.transformers_version import ( _resolve_base_model, _check_tokenizer_config_needs_v5, + _check_config_needs_550, _tokenizer_class_cache, + _config_needs_550_cache, needs_transformers_5, + get_transformers_tier, ) @@ -188,3 +191,148 @@ class TestNeedsTransformers5: # We test the full resolution chain here: resolved = _resolve_base_model(str(tmp_path)) assert needs_transformers_5(resolved) is True + + +# --------------------------------------------------------------------------- +# _check_config_needs_550 — config.json architecture/model_type check +# --------------------------------------------------------------------------- + + +class TestCheckConfigNeeds550: + """Tests for _check_config_needs_550() local config.json checks.""" + + def setup_method(self): + _config_needs_550_cache.clear() + + def test_gemma4_architecture(self, tmp_path: Path): + """config.json with Gemma4ForConditionalGeneration should return True.""" + cfg = { + "architectures": ["Gemma4ForConditionalGeneration"], + "model_type": "gemma4", + } + (tmp_path / "config.json").write_text(json.dumps(cfg)) + + assert _check_config_needs_550(str(tmp_path)) is True + + def test_gemma4_model_type_only(self, tmp_path: Path): + """config.json with model_type=gemma4 (no architectures) should return True.""" + cfg = {"model_type": "gemma4"} + (tmp_path / "config.json").write_text(json.dumps(cfg)) + + assert _check_config_needs_550(str(tmp_path)) is True + + def test_llama_architecture(self, tmp_path: Path): + """config.json with LlamaForCausalLM should return False.""" + cfg = {"architectures": ["LlamaForCausalLM"], "model_type": "llama"} + (tmp_path / "config.json").write_text(json.dumps(cfg)) + + assert _check_config_needs_550(str(tmp_path)) is False + + def test_no_config_json(self, tmp_path: Path): + """Missing config.json should return False (fail-open).""" + # Patch network call to avoid real fetch + with patch("urllib.request.urlopen") as mock_urlopen: + mock_urlopen.side_effect = Exception("no network") + assert _check_config_needs_550(str(tmp_path)) is False + + def test_result_is_cached(self, tmp_path: Path): + """Subsequent calls should use the cache.""" + cfg = {"architectures": ["Gemma4ForConditionalGeneration"]} + (tmp_path / "config.json").write_text(json.dumps(cfg)) + + key = str(tmp_path) + _check_config_needs_550(key) + assert key in _config_needs_550_cache + assert _config_needs_550_cache[key] is True + + def test_local_file_skips_network(self, tmp_path: Path): + """When local config.json exists, no network request should be made.""" + cfg = {"architectures": ["LlamaForCausalLM"]} + (tmp_path / "config.json").write_text(json.dumps(cfg)) + + with patch("urllib.request.urlopen") as mock_urlopen: + _check_config_needs_550(str(tmp_path)) + mock_urlopen.assert_not_called() + + +# --------------------------------------------------------------------------- +# get_transformers_tier — tier detection +# --------------------------------------------------------------------------- + + +class TestGetTransformersTier: + """Tests for get_transformers_tier() tiered version detection.""" + + def setup_method(self): + _tokenizer_class_cache.clear() + _config_needs_550_cache.clear() + + def test_gemma4_substring_returns_550(self): + assert get_transformers_tier("google/gemma-4-E2B-it") == "550" + + def test_gemma4_alt_substring_returns_550(self): + assert get_transformers_tier("unsloth/gemma4-E4B-it") == "550" + + def test_gemma4_config_json_returns_550(self, tmp_path: Path): + """Local checkpoint with Gemma4 architecture → 550.""" + cfg = { + "architectures": ["Gemma4ForConditionalGeneration"], + "model_type": "gemma4", + } + (tmp_path / "config.json").write_text(json.dumps(cfg)) + + assert get_transformers_tier(str(tmp_path)) == "550" + + def test_qwen35_returns_530(self): + with patch( + "utils.transformers_version._check_config_needs_550", + return_value = False, + ): + assert get_transformers_tier("Qwen/Qwen3.5-9B") == "530" + + def test_ministral_returns_530(self): + with patch( + "utils.transformers_version._check_config_needs_550", + return_value = False, + ): + assert ( + get_transformers_tier("mistralai/Ministral-3-8B-Instruct-2512") == "530" + ) + + def test_llama_returns_default(self): + with ( + patch( + "utils.transformers_version._check_config_needs_550", + return_value = False, + ), + patch( + "utils.transformers_version._check_tokenizer_config_needs_v5", + return_value = False, + ), + ): + assert get_transformers_tier("meta-llama/Llama-3-8B") == "default" + + def test_550_checked_before_530(self): + """Ensure 5.5.0 is checked first — a model matching both should get 550.""" + # This shouldn't happen in practice, but verifies priority + assert get_transformers_tier("gemma-4-model") == "550" + + def test_needs_transformers_5_compat(self): + """needs_transformers_5 should return True for both 530 and 550 models.""" + assert needs_transformers_5("google/gemma-4-E2B-it") is True + with patch( + "utils.transformers_version._check_config_needs_550", + return_value = False, + ): + assert needs_transformers_5("Qwen/Qwen3.5-9B") is True + with ( + patch( + "utils.transformers_version._check_config_needs_550", + return_value = False, + ), + patch( + "utils.transformers_version._check_tokenizer_config_needs_v5", + return_value = False, + ), + ): + assert needs_transformers_5("meta-llama/Llama-3-8B") is False diff --git a/studio/backend/utils/models/model_config.py b/studio/backend/utils/models/model_config.py index 61226e52cb..5be0b183d5 100644 --- a/studio/backend/utils/models/model_config.py +++ b/studio/backend/utils/models/model_config.py @@ -493,8 +493,9 @@ _VLM_MODEL_TYPES = { "minicpmv", } -# Pre-computed .venv_t5 path and backend dir for subprocess version switching. -_VENV_T5_DIR = str(Path.home() / ".unsloth" / "studio" / ".venv_t5") +# Pre-computed .venv_t5 paths and backend dir for subprocess version switching. +# Vision check uses 5.5.0 (newest, recognizes all architectures). +_VENV_T5_DIR = str(Path.home() / ".unsloth" / "studio" / ".venv_t5_550") _BACKEND_DIR = str(Path(__file__).resolve().parent.parent.parent) # Inline script executed in a subprocess with transformers 5.x activated. diff --git a/studio/backend/utils/transformers_version.py b/studio/backend/utils/transformers_version.py index 07e4a5c000..0c13b5455b 100644 --- a/studio/backend/utils/transformers_version.py +++ b/studio/backend/utils/transformers_version.py @@ -5,20 +5,25 @@ Automatic transformers version switching. Some newer model architectures (Ministral-3, GLM-4.7-Flash, Qwen3-30B-A3B MoE, -tiny_qwen3_moe) require transformers>=5.3.0, while everything else needs the -default 4.57.x that ships with Unsloth. +tiny_qwen3_moe) require transformers>=5.3.0, while Gemma 4 models require +transformers>=5.5.0. Everything else needs the default 4.57.x that ships +with Unsloth. + +Two separate target directories are maintained: + - .venv_t5_530/ — transformers 5.3.0 (Ministral-3, GLM, Qwen3 MoE, etc.) + - .venv_t5_550/ — transformers 5.5.0 (Gemma 4) When loading a LoRA adapter with a custom name, we resolve the base model from ``adapter_config.json`` and check *that* against the model list. Strategy: Training and inference run in subprocesses that activate the correct version - via sys.path (prepending .venv_t5/ for 5.x models). See: + via sys.path (prepending the appropriate .venv_t5_*/ directory). See: - core/training/worker.py - core/inference/worker.py For export (still in-process), ensure_transformers_version() does a lightweight - sys.path swap using the same .venv_t5/ directory pre-installed by setup.sh. + sys.path swap using the same directories pre-installed by setup.sh. """ import importlib @@ -39,7 +44,7 @@ logger = get_logger(__name__) # --------------------------------------------------------------------------- # Lowercase substrings — if ANY appears anywhere in the lowered model name, -# we need transformers 5.x. +# we need transformers 5.3.0. TRANSFORMERS_5_MODEL_SUBSTRINGS: tuple[str, ...] = ( "ministral-3-", # Ministral-3-{3,8,14}B-{Instruct,Reasoning,Base}-2512 "glm-4.7-flash", # GLM-4.7-Flash @@ -47,10 +52,23 @@ TRANSFORMERS_5_MODEL_SUBSTRINGS: tuple[str, ...] = ( "qwen3.5", # Qwen3.5 family (35B-A3B, etc.) "qwen3-next", # Qwen3-Next and variants "tiny_qwen3_moe", # imdatta0/tiny_qwen3_moe_2.8B_0.7B +) + +# Lowercase substrings for models that require transformers 5.5.0 (checked first). +TRANSFORMERS_550_MODEL_SUBSTRINGS: tuple[str, ...] = ( "gemma-4", # Gemma-4 (E2B-it, E4B-it, 31B-it, 26B-A4B-it) "gemma4", # Gemma-4 alternate naming ) +# Architecture classes / model_type values that require transformers 5.5.0. +# Checked via config.json (local or HuggingFace). +_TRANSFORMERS_550_ARCHITECTURES: set[str] = { + "Gemma4ForConditionalGeneration", +} +_TRANSFORMERS_550_MODEL_TYPES: set[str] = { + "gemma4", +} + # Tokenizer classes that only exist in transformers>=5.x _TRANSFORMERS_5_TOKENIZER_CLASSES: set[str] = { "TokenizersBackend", @@ -59,12 +77,61 @@ _TRANSFORMERS_5_TOKENIZER_CLASSES: set[str] = { # Cache for dynamic tokenizer_config.json lookups to avoid repeated fetches _tokenizer_class_cache: dict[str, bool] = {} -# Versions -TRANSFORMERS_5_VERSION = "5.5.0" -TRANSFORMERS_DEFAULT_VERSION = "4.57.6" +# Cache for dynamic config.json lookups (architecture/model_type checks) +_config_needs_550_cache: dict[str, bool] = {} -# Pre-installed directory for transformers 5.x — created by setup.sh / setup.ps1 -_VENV_T5_DIR = str(Path.home() / ".unsloth" / "studio" / ".venv_t5") +# Versions +TRANSFORMERS_550_VERSION = "5.5.0" +TRANSFORMERS_530_VERSION = "5.3.0" +TRANSFORMERS_DEFAULT_VERSION = "4.57.6" +# Backwards-compat alias — points to 5.5.0 (the highest 5.x tier). +# Consumers should prefer TRANSFORMERS_530_VERSION / TRANSFORMERS_550_VERSION. +TRANSFORMERS_5_VERSION = TRANSFORMERS_550_VERSION + +# Pre-installed directories — created by setup.sh / setup.ps1 +_VENV_T5_530_DIR = str(Path.home() / ".unsloth" / "studio" / ".venv_t5_530") +_VENV_T5_550_DIR = str(Path.home() / ".unsloth" / "studio" / ".venv_t5_550") +# Backwards-compat alias +_VENV_T5_DIR = _VENV_T5_550_DIR + + +def activate_transformers_for_subprocess(model_name: str) -> None: + """Activate the correct transformers version in a subprocess worker. + + Call this BEFORE any ML imports. Resolves LoRA adapters to their base + model, determines the required tier, and prepends the appropriate + ``.venv_t5_*`` directory to ``sys.path``. Also propagates the path + via ``PYTHONPATH`` for child processes (e.g. GGUF converter). + + Used by training, inference, and export workers. + """ + resolved = _resolve_base_model(model_name) + tier = get_transformers_tier(resolved) + + if tier == "550": + if not _ensure_venv_t5_550_exists(): + raise RuntimeError( + f"Cannot activate transformers 5.5.0: " + f".venv_t5_550 missing at {_VENV_T5_550_DIR}" + ) + if _VENV_T5_550_DIR not in sys.path: + sys.path.insert(0, _VENV_T5_550_DIR) + logger.info("Activated transformers 5.5.0 from %s", _VENV_T5_550_DIR) + _pp = os.environ.get("PYTHONPATH", "") + os.environ["PYTHONPATH"] = _VENV_T5_550_DIR + (os.pathsep + _pp if _pp else "") + elif tier == "530": + if not _ensure_venv_t5_530_exists(): + raise RuntimeError( + f"Cannot activate transformers 5.3.0: " + f".venv_t5_530 missing at {_VENV_T5_530_DIR}" + ) + if _VENV_T5_530_DIR not in sys.path: + sys.path.insert(0, _VENV_T5_530_DIR) + logger.info("Activated transformers 5.3.0 from %s", _VENV_T5_530_DIR) + _pp = os.environ.get("PYTHONPATH", "") + os.environ["PYTHONPATH"] = _VENV_T5_530_DIR + (os.pathsep + _pp if _pp else "") + else: + logger.info("Using default transformers (4.57.x) for %s", model_name) def _resolve_base_model(model_name: str) -> str: @@ -192,18 +259,103 @@ def _check_tokenizer_config_needs_v5(model_name: str) -> bool: return False -def needs_transformers_5(model_name: str) -> bool: - """Return True if *model_name* belongs to an architecture that requires - ``transformers>=5.3.0``. +def _check_config_needs_550(model_name: str) -> bool: + """Check ``config.json`` for architectures or model_type that require + transformers 5.5.0 (e.g. Gemma 4). - First checks the hardcoded substring list for known models, then - dynamically fetches ``tokenizer_config.json`` from HuggingFace to check - if the tokenizer_class (e.g. ``TokenizersBackend``) requires v5. + Checks locally first, then falls back to fetching from HuggingFace. + Results are cached in ``_config_needs_550_cache``. + Returns False on any error (fail-open to lower tier). + """ + if model_name in _config_needs_550_cache: + return _config_needs_550_cache[model_name] + + def _check_cfg(cfg: dict) -> bool: + archs = cfg.get("architectures", []) + if any(a in _TRANSFORMERS_550_ARCHITECTURES for a in archs): + return True + if cfg.get("model_type") in _TRANSFORMERS_550_MODEL_TYPES: + return True + return False + + # --- Check local config.json first ------------------------------------ + local_path = Path(model_name) + local_cfg = local_path / "config.json" + if local_cfg.is_file(): + try: + with open(local_cfg) as f: + cfg = json.load(f) + result = _check_cfg(cfg) + if result: + logger.info( + "Local config.json check: %s needs transformers 5.5.0 " + "(architectures=%s, model_type=%s)", + model_name, + cfg.get("architectures", []), + cfg.get("model_type"), + ) + _config_needs_550_cache[model_name] = result + return result + except Exception as exc: + logger.debug("Could not read %s: %s", local_cfg, exc) + + # --- Fall back to fetching from HuggingFace --------------------------- + import urllib.request + + url = f"https://huggingface.co/{model_name}/raw/main/config.json" + try: + req = urllib.request.Request(url, headers = {"User-Agent": "unsloth-studio"}) + with urllib.request.urlopen(req, timeout = 10) as resp: + cfg = json.loads(resp.read().decode()) + result = _check_cfg(cfg) + if result: + logger.info( + "Dynamic config.json check: %s needs transformers 5.5.0 " + "(architectures=%s, model_type=%s)", + model_name, + cfg.get("architectures", []), + cfg.get("model_type"), + ) + _config_needs_550_cache[model_name] = result + return result + except Exception as exc: + logger.debug("Could not fetch config.json for '%s': %s", model_name, exc) + _config_needs_550_cache[model_name] = False + return False + + +def get_transformers_tier(model_name: str) -> str: + """Return the transformers tier required for *model_name*. + + Returns ``"550"`` for models needing transformers 5.5.0 (e.g. Gemma 4), + ``"530"`` for models needing transformers 5.3.0 (e.g. Ministral-3, Qwen3 MoE), + or ``"default"`` for everything else (4.57.x). + + The 5.5.0 check runs first, then 5.3.0. """ lowered = model_name.lower() + + # --- Fast substring checks (no I/O) ------------------------------------ + if any(sub in lowered for sub in TRANSFORMERS_550_MODEL_SUBSTRINGS): + return "550" if any(sub in lowered for sub in TRANSFORMERS_5_MODEL_SUBSTRINGS): - return True - return _check_tokenizer_config_needs_v5(model_name) + return "530" + + # --- Slow config fallbacks (local file first, then network) ----------- + if _check_config_needs_550(model_name): + return "550" + if _check_tokenizer_config_needs_v5(model_name): + return "530" + + return "default" + + +def needs_transformers_5(model_name: str) -> bool: + """Return True if *model_name* requires any transformers 5.x version. + + Convenience wrapper around :func:`get_transformers_tier`. + """ + return get_transformers_tier(model_name) != "default" # --------------------------------------------------------------------------- @@ -258,27 +410,36 @@ def _purge_modules() -> int: return len(to_remove) -_VENV_T5_PACKAGES = ( - f"transformers=={TRANSFORMERS_5_VERSION}", +_VENV_T5_530_PACKAGES = ( + f"transformers=={TRANSFORMERS_530_VERSION}", "huggingface_hub==1.8.0", "hf_xet==1.4.2", "tiktoken", ) +_VENV_T5_550_PACKAGES = ( + f"transformers=={TRANSFORMERS_550_VERSION}", + "huggingface_hub==1.8.0", + "hf_xet==1.4.2", + "tiktoken", +) -def _venv_t5_is_valid() -> bool: - """Return True if .venv_t5/ has all required packages at the correct versions.""" - if not os.path.isdir(_VENV_T5_DIR) or not os.listdir(_VENV_T5_DIR): +# Backwards-compat alias +_VENV_T5_PACKAGES = _VENV_T5_550_PACKAGES + + +def _venv_dir_is_valid(venv_dir: str, packages: tuple[str, ...]) -> bool: + """Return True if *venv_dir* has all *packages* at the correct versions.""" + if not os.path.isdir(venv_dir) or not os.listdir(venv_dir): return False - # Check that the key package directories exist AND match the required version - for pkg_spec in _VENV_T5_PACKAGES: + for pkg_spec in packages: parts = pkg_spec.split("==") pkg_name = parts[0] pkg_version = parts[1] if len(parts) > 1 else None pkg_name_norm = pkg_name.replace("-", "_") # Check directory exists if not any( - (Path(_VENV_T5_DIR) / d).is_dir() + (Path(venv_dir) / d).is_dir() for d in (pkg_name_norm, pkg_name_norm.replace("_", "-")) ): return False @@ -287,7 +448,7 @@ def _venv_t5_is_valid() -> bool: continue # Check version via .dist-info metadata dist_info_found = False - for di in Path(_VENV_T5_DIR).glob(f"{pkg_name_norm}-*.dist-info"): + for di in Path(venv_dir).glob(f"{pkg_name_norm}-*.dist-info"): metadata = di / "METADATA" if not metadata.is_file(): continue @@ -296,7 +457,8 @@ def _venv_t5_is_valid() -> bool: installed_ver = line.split(":", 1)[1].strip() if installed_ver != pkg_version: logger.info( - ".venv_t5 has %s==%s but need %s", + "%s has %s==%s but need %s", + venv_dir, pkg_name, installed_ver, pkg_version, @@ -311,8 +473,13 @@ def _venv_t5_is_valid() -> bool: return True -def _install_to_venv_t5(pkg: str) -> bool: - """Install a single package into .venv_t5/, preferring uv then pip.""" +def _venv_t5_is_valid() -> bool: + """Backwards-compat: check the 5.5.0 venv.""" + return _venv_dir_is_valid(_VENV_T5_550_DIR, _VENV_T5_550_PACKAGES) + + +def _install_to_dir(pkg: str, target_dir: str) -> bool: + """Install a single package into *target_dir*, preferring uv then pip.""" # Try uv first (faster) if already on PATH -- do NOT install uv at runtime if shutil.which("uv"): result = subprocess.run( @@ -323,7 +490,7 @@ def _install_to_venv_t5(pkg: str) -> bool: "--python", sys.executable, "--target", - _VENV_T5_DIR, + target_dir, "--no-deps", "--upgrade", pkg, @@ -344,7 +511,7 @@ def _install_to_venv_t5(pkg: str) -> bool: "pip", "install", "--target", - _VENV_T5_DIR, + target_dir, "--no-deps", "--upgrade", pkg, @@ -359,47 +526,62 @@ def _install_to_venv_t5(pkg: str) -> bool: return True -def _ensure_venv_t5_exists() -> bool: - """Ensure .venv_t5/ exists with all required packages. Install if missing.""" - if _venv_t5_is_valid(): +def _ensure_venv_dir(venv_dir: str, packages: tuple[str, ...], label: str) -> bool: + """Ensure *venv_dir* exists with all *packages*. Install if missing.""" + if _venv_dir_is_valid(venv_dir, packages): return True logger.warning( - ".venv_t5 not found or incomplete at %s -- installing at runtime", _VENV_T5_DIR + "%s not found or incomplete at %s -- installing at runtime", label, venv_dir ) - shutil.rmtree(_VENV_T5_DIR, ignore_errors = True) - os.makedirs(_VENV_T5_DIR, exist_ok = True) - for pkg in _VENV_T5_PACKAGES: - if not _install_to_venv_t5(pkg): + shutil.rmtree(venv_dir, ignore_errors = True) + os.makedirs(venv_dir, exist_ok = True) + for pkg in packages: + if not _install_to_dir(pkg, venv_dir): return False - logger.info("Installed transformers 5.x to %s", _VENV_T5_DIR) + logger.info("Installed %s to %s", label, venv_dir) return True -def _activate_5x() -> None: - """Prepend .venv_t5/ to sys.path, purge stale modules, reimport.""" - if not _ensure_venv_t5_exists(): - raise RuntimeError( - f"Cannot activate transformers 5.x: .venv_t5 missing at {_VENV_T5_DIR}" - ) +def _ensure_venv_t5_530_exists() -> bool: + """Ensure .venv_t5_530/ exists with transformers 5.3.0.""" + return _ensure_venv_dir( + _VENV_T5_530_DIR, _VENV_T5_530_PACKAGES, "transformers 5.3.0" + ) - if _VENV_T5_DIR not in sys.path: - sys.path.insert(0, _VENV_T5_DIR) - logger.info("Prepended %s to sys.path", _VENV_T5_DIR) + +def _ensure_venv_t5_550_exists() -> bool: + """Ensure .venv_t5_550/ exists with transformers 5.5.0.""" + return _ensure_venv_dir( + _VENV_T5_550_DIR, _VENV_T5_550_PACKAGES, "transformers 5.5.0" + ) + + +def _ensure_venv_t5_exists() -> bool: + """Backwards-compat: ensure the 5.5.0 venv exists.""" + return _ensure_venv_t5_550_exists() + + +def _activate_venv(venv_dir: str, label: str) -> None: + """Prepend *venv_dir* to sys.path, purge stale modules, reimport.""" + if venv_dir not in sys.path: + sys.path.insert(0, venv_dir) + logger.info("Prepended %s to sys.path", venv_dir) count = _purge_modules() logger.info("Purged %d cached modules", count) import transformers - logger.info("Loaded transformers %s", transformers.__version__) + logger.info("Loaded transformers %s (%s)", transformers.__version__, label) def _deactivate_5x() -> None: - """Remove .venv_t5/ from sys.path, purge stale modules, reimport.""" - while _VENV_T5_DIR in sys.path: - sys.path.remove(_VENV_T5_DIR) - logger.info("Removed %s from sys.path", _VENV_T5_DIR) + """Remove all .venv_t5_*/ dirs from sys.path, purge stale modules, reimport.""" + for d in (_VENV_T5_530_DIR, _VENV_T5_550_DIR): + while d in sys.path: + sys.path.remove(d) + logger.info("Removed venv_t5 dirs from sys.path") count = _purge_modules() logger.info("Purged %d cached modules", count) @@ -412,9 +594,10 @@ def _deactivate_5x() -> None: def ensure_transformers_version(model_name: str) -> None: """Ensure the correct ``transformers`` version is active for *model_name*. - Uses sys.path with .venv_t5/ (pre-installed by setup.sh): - • Need 5.x → prepend .venv_t5/ to sys.path, purge modules. - • Need 4.x → remove .venv_t5/ from sys.path, purge modules. + Uses sys.path with .venv_t5_530/ or .venv_t5_550/ (pre-installed by setup.sh): + • Need 5.5.0 → prepend .venv_t5_550/ to sys.path, purge modules. + • Need 5.3.0 → prepend .venv_t5_530/ to sys.path, purge modules. + • Need 4.x → remove all .venv_t5_*/ from sys.path, purge modules. For LoRA adapters with custom names, the base model is resolved from ``adapter_config.json`` before checking. @@ -424,8 +607,21 @@ def ensure_transformers_version(model_name: str) -> None: """ # Resolve LoRA adapters to their base model for accurate detection resolved = _resolve_base_model(model_name) - want_5 = needs_transformers_5(resolved) - target_version = TRANSFORMERS_5_VERSION if want_5 else TRANSFORMERS_DEFAULT_VERSION + tier = get_transformers_tier(resolved) + + if tier == "550": + target_version = TRANSFORMERS_550_VERSION + venv_dir = _VENV_T5_550_DIR + ensure_fn = _ensure_venv_t5_550_exists + elif tier == "530": + target_version = TRANSFORMERS_530_VERSION + venv_dir = _VENV_T5_530_DIR + ensure_fn = _ensure_venv_t5_530_exists + else: + target_version = TRANSFORMERS_DEFAULT_VERSION + venv_dir = None + ensure_fn = None + target_major = int(target_version.split(".")[0]) # Check what's actually loaded in memory @@ -441,8 +637,17 @@ def ensure_transformers_version(model_name: str) -> None: # --- Already correct? --------------------------------------------------- if in_memory is not None: + if in_memory == target_version: + logger.info( + "transformers %s already loaded — correct for '%s'", + in_memory, + model_name, + ) + return + # Different 5.x → need to switch (e.g. 5.3.0 loaded but need 5.5.0) in_memory_major = int(in_memory.split(".")[0]) - if in_memory_major == target_major: + if in_memory_major == target_major and venv_dir is None: + # Both are default (4.x) — close enough logger.info( "transformers %s already loaded — correct for '%s'", in_memory, @@ -451,9 +656,16 @@ def ensure_transformers_version(model_name: str) -> None: return # --- Switch version ----------------------------------------------------- - if want_5: - logger.info("Activating transformers %s via .venv_t5…", TRANSFORMERS_5_VERSION) - _activate_5x() + if venv_dir is not None: + # First remove any other 5.x venv from sys.path + _deactivate_5x() + if not ensure_fn(): + raise RuntimeError( + f"Cannot activate transformers {target_version}: " + f"venv missing at {venv_dir}" + ) + logger.info("Activating transformers %s…", target_version) + _activate_venv(venv_dir, f"transformers {target_version}") else: logger.info( "Reverting to default transformers %s…", TRANSFORMERS_DEFAULT_VERSION diff --git a/studio/setup.ps1 b/studio/setup.ps1 index 60dfce0661..c3a8cd71ca 100644 --- a/studio/setup.ps1 +++ b/studio/setup.ps1 @@ -1579,55 +1579,104 @@ if ($stackExit -ne 0) { exit 1 } -# ── Pre-install transformers 5.x into .venv_t5/ ── -# Models like GLM-4.7-Flash need transformers>=5.3.0. Instead of pip-installing -# at runtime (slow, ~10-15s), we pre-install into a separate directory. -# The training subprocess just prepends .venv_t5/ to sys.path -- instant switch. -Write-Host "" -substep "pre-installing transformers 5.x for newer model support..." -$VenvT5Dir = Join-Path $env:USERPROFILE ".unsloth\studio\.venv_t5" -if (Test-Path $VenvT5Dir) { Remove-Item -Recurse -Force $VenvT5Dir } -New-Item -ItemType Directory -Path $VenvT5Dir -Force | Out-Null -$prevEAP_t5 = $ErrorActionPreference -$ErrorActionPreference = "Continue" -foreach ($pkg in @("transformers==5.5.0", "huggingface_hub==1.8.0", "hf_xet==1.4.2")) { - if ($script:UnslothVerbose) { - Fast-Install --target $VenvT5Dir --no-deps $pkg - $t5PkgExit = $LASTEXITCODE - $output = "" - } else { - $output = Fast-Install --target $VenvT5Dir --no-deps $pkg | Out-String - $t5PkgExit = $LASTEXITCODE - } - if ($t5PkgExit -ne 0) { - Write-Host "[FAIL] Could not install $pkg into .venv_t5/" -ForegroundColor Red - Write-Host $output -ForegroundColor Red - $ErrorActionPreference = $prevEAP_t5 - exit 1 - } -} -# tiktoken is needed by Qwen-family tokenizers -- install with deps since -# regex/requests may be missing on Windows -if ($script:UnslothVerbose) { - Fast-Install --target $VenvT5Dir tiktoken - $tiktokenInstallExit = $LASTEXITCODE - $output = "" -} else { - $output = Fast-Install --target $VenvT5Dir tiktoken | Out-String - $tiktokenInstallExit = $LASTEXITCODE -} -if ($tiktokenInstallExit -ne 0) { - substep "Could not install tiktoken into .venv_t5/ -- Qwen tokenizers may fail" "Yellow" -} -$ErrorActionPreference = $prevEAP_t5 -step "transformers" "5.x pre-installed" - } else { step "python" "dependencies up to date" # Restore ErrorActionPreference (was lowered for pip/python section) $ErrorActionPreference = $prevEAP } +# ── Pre-install transformers 5.x into .venv_t5_530/ and .venv_t5_550/ ── +# Runs outside the deps fast-path gate so that upgrades from the legacy +# single .venv_t5 are always migrated to the tiered layout. +$VenvT5_530Dir = Join-Path $env:USERPROFILE ".unsloth\studio\.venv_t5_530" +$VenvT5_550Dir = Join-Path $env:USERPROFILE ".unsloth\studio\.venv_t5_550" +$VenvT5Legacy = Join-Path $env:USERPROFILE ".unsloth\studio\.venv_t5" + +$_NeedT5Install = $false +if (Test-Path $VenvT5Legacy) { + Remove-Item -Recurse -Force $VenvT5Legacy + $_NeedT5Install = $true +} +if (-not (Test-Path $VenvT5_530Dir)) { $_NeedT5Install = $true } +if (-not (Test-Path $VenvT5_550Dir)) { $_NeedT5Install = $true } +# Also reinstall when python deps were updated +if (-not $SkipPythonDeps) { $_NeedT5Install = $true } + +if ($_NeedT5Install) { +Write-Host "" + +$prevEAP_t5 = $ErrorActionPreference +$ErrorActionPreference = "Continue" + +# --- .venv_t5_530 (transformers 5.3.0) --- +substep "pre-installing transformers 5.3.0 for newer model support..." +if (Test-Path $VenvT5_530Dir) { Remove-Item -Recurse -Force $VenvT5_530Dir } +New-Item -ItemType Directory -Path $VenvT5_530Dir -Force | Out-Null +foreach ($pkg in @("transformers==5.3.0", "huggingface_hub==1.8.0", "hf_xet==1.4.2")) { + if ($script:UnslothVerbose) { + Fast-Install --target $VenvT5_530Dir --no-deps $pkg + $t5PkgExit = $LASTEXITCODE + $output = "" + } else { + $output = Fast-Install --target $VenvT5_530Dir --no-deps $pkg | Out-String + $t5PkgExit = $LASTEXITCODE + } + if ($t5PkgExit -ne 0) { + Write-Host "[FAIL] Could not install $pkg into .venv_t5_530/" -ForegroundColor Red + Write-Host $output -ForegroundColor Red + $ErrorActionPreference = $prevEAP_t5 + exit 1 + } +} +if ($script:UnslothVerbose) { + Fast-Install --target $VenvT5_530Dir tiktoken + $tiktokenInstallExit = $LASTEXITCODE + $output = "" +} else { + $output = Fast-Install --target $VenvT5_530Dir tiktoken | Out-String + $tiktokenInstallExit = $LASTEXITCODE +} +if ($tiktokenInstallExit -ne 0) { + substep "Could not install tiktoken into .venv_t5_530/ -- Qwen tokenizers may fail" "Yellow" +} +step "transformers" "5.3.0 pre-installed" + +# --- .venv_t5_550 (transformers 5.5.0) --- +substep "pre-installing transformers 5.5.0 for Gemma 4 support..." +if (Test-Path $VenvT5_550Dir) { Remove-Item -Recurse -Force $VenvT5_550Dir } +New-Item -ItemType Directory -Path $VenvT5_550Dir -Force | Out-Null +foreach ($pkg in @("transformers==5.5.0", "huggingface_hub==1.8.0", "hf_xet==1.4.2")) { + if ($script:UnslothVerbose) { + Fast-Install --target $VenvT5_550Dir --no-deps $pkg + $t5PkgExit = $LASTEXITCODE + $output = "" + } else { + $output = Fast-Install --target $VenvT5_550Dir --no-deps $pkg | Out-String + $t5PkgExit = $LASTEXITCODE + } + if ($t5PkgExit -ne 0) { + Write-Host "[FAIL] Could not install $pkg into .venv_t5_550/" -ForegroundColor Red + Write-Host $output -ForegroundColor Red + $ErrorActionPreference = $prevEAP_t5 + exit 1 + } +} +if ($script:UnslothVerbose) { + Fast-Install --target $VenvT5_550Dir tiktoken + $tiktokenInstallExit = $LASTEXITCODE + $output = "" +} else { + $output = Fast-Install --target $VenvT5_550Dir tiktoken | Out-String + $tiktokenInstallExit = $LASTEXITCODE +} +if ($tiktokenInstallExit -ne 0) { + substep "Could not install tiktoken into .venv_t5_550/ -- Qwen tokenizers may fail" "Yellow" +} +$ErrorActionPreference = $prevEAP_t5 +step "transformers" "5.5.0 pre-installed" + +} # end $_NeedT5Install + # ========================================================================== # PHASE 3.4: Prefer prebuilt llama.cpp bundles before source build # ========================================================================== diff --git a/studio/setup.sh b/studio/setup.sh index e3ff2da35c..eb89e583ca 100755 --- a/studio/setup.sh +++ b/studio/setup.sh @@ -394,11 +394,14 @@ fi # ── Python venv + deps ── STUDIO_HOME="$HOME/.unsloth/studio" VENV_DIR="$STUDIO_HOME/unsloth_studio" -VENV_T5_DIR="$STUDIO_HOME/.venv_t5" +VENV_T5_530_DIR="$STUDIO_HOME/.venv_t5_530" +VENV_T5_550_DIR="$STUDIO_HOME/.venv_t5_550" [ -d "$REPO_ROOT/.venv" ] && rm -rf "$REPO_ROOT/.venv" [ -d "$REPO_ROOT/.venv_overlay" ] && rm -rf "$REPO_ROOT/.venv_overlay" [ -d "$REPO_ROOT/.venv_t5" ] && rm -rf "$REPO_ROOT/.venv_t5" +[ -d "$REPO_ROOT/.venv_t5_530" ] && rm -rf "$REPO_ROOT/.venv_t5_530" +[ -d "$REPO_ROOT/.venv_t5_550" ] && rm -rf "$REPO_ROOT/.venv_t5_550" # Note: do NOT delete $STUDIO_HOME/.venv here — install.sh handles migration _COLAB_NO_VENV=false @@ -501,21 +504,47 @@ fi if [ "$_SKIP_PYTHON_DEPS" = false ]; then install_python_stack - - # ── 6b. Pre-install transformers 5.x into .venv_t5/ ── - # Models like GLM-4.7-Flash need transformers>=5.3.0. Instead of pip-installing - # at runtime (slow, ~10-15s), we pre-install into a separate directory. - # The training subprocess just prepends .venv_t5/ to sys.path -- instant switch. - mkdir -p "$VENV_T5_DIR" - run_quiet "install transformers 5.x" fast_install --target "$VENV_T5_DIR" --no-deps "transformers==5.5.0" - run_quiet "install huggingface_hub for t5" fast_install --target "$VENV_T5_DIR" --no-deps "huggingface_hub==1.8.0" - run_quiet "install hf_xet for t5" fast_install --target "$VENV_T5_DIR" --no-deps "hf_xet==1.4.2" - run_quiet "install tiktoken for t5" fast_install --target "$VENV_T5_DIR" "tiktoken" - step "transformers" "5.x pre-installed" else step "python" "dependencies up to date" verbose_substep "python deps check: installed=$_PKG_NAME@${INSTALLED_VER:-unknown} latest=${LATEST_VER:-unknown}" fi + +# ── 6b. Pre-install transformers 5.x into .venv_t5_530/ and .venv_t5_550/ ── +# Models like GLM-4.7-Flash, Qwen3 MoE need transformers>=5.3.0. +# Gemma 4 models need transformers>=5.5.0. +# Pre-install into separate directories to avoid runtime pip overhead. +# The training subprocess prepends the appropriate dir to sys.path. +# +# Runs outside the _SKIP_PYTHON_DEPS gate so that upgrades from legacy +# single .venv_t5 are always migrated to the tiered layout. +_NEED_T5_INSTALL=false +if [ -d "$STUDIO_HOME/.venv_t5" ]; then + # Legacy layout — migrate + rm -rf "$STUDIO_HOME/.venv_t5" + _NEED_T5_INSTALL=true +fi +[ ! -d "$VENV_T5_530_DIR" ] && _NEED_T5_INSTALL=true +[ ! -d "$VENV_T5_550_DIR" ] && _NEED_T5_INSTALL=true +# Also reinstall when python deps were updated (packages may need rebuild) +[ "$_SKIP_PYTHON_DEPS" = false ] && _NEED_T5_INSTALL=true + +if [ "$_NEED_T5_INSTALL" = true ]; then + [ -d "$VENV_T5_530_DIR" ] && rm -rf "$VENV_T5_530_DIR" + mkdir -p "$VENV_T5_530_DIR" + run_quiet "install transformers 5.3.0" fast_install --target "$VENV_T5_530_DIR" --no-deps "transformers==5.3.0" + run_quiet "install huggingface_hub for t5_530" fast_install --target "$VENV_T5_530_DIR" --no-deps "huggingface_hub==1.8.0" + run_quiet "install hf_xet for t5_530" fast_install --target "$VENV_T5_530_DIR" --no-deps "hf_xet==1.4.2" + run_quiet "install tiktoken for t5_530" fast_install --target "$VENV_T5_530_DIR" "tiktoken" + step "transformers" "5.3.0 pre-installed" + + [ -d "$VENV_T5_550_DIR" ] && rm -rf "$VENV_T5_550_DIR" + mkdir -p "$VENV_T5_550_DIR" + run_quiet "install transformers 5.5.0" fast_install --target "$VENV_T5_550_DIR" --no-deps "transformers==5.5.0" + run_quiet "install huggingface_hub for t5_550" fast_install --target "$VENV_T5_550_DIR" --no-deps "huggingface_hub==1.8.0" + run_quiet "install hf_xet for t5_550" fast_install --target "$VENV_T5_550_DIR" --no-deps "hf_xet==1.4.2" + run_quiet "install tiktoken for t5_550" fast_install --target "$VENV_T5_550_DIR" "tiktoken" + step "transformers" "5.5.0 pre-installed" +fi fi # ── 7. Prefer prebuilt llama.cpp bundles before any source build path ── From c2184af07985b30eecdd41480d52dfafefc1e2ec Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 7 Apr 2026 22:50:48 -0700 Subject: [PATCH 03/60] [pre-commit.ci] pre-commit autoupdate (#4879) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.15.8 → v0.15.9](https://github.com/astral-sh/ruff-pre-commit/compare/v0.15.8...v0.15.9) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f3b5987a9c..e41c37d209 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.15.8 + rev: v0.15.9 hooks: - id: ruff args: From 67e9db4921e16562497dfd5c873e8bd01970906b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 8 Apr 2026 03:35:33 -0700 Subject: [PATCH 04/60] build(deps): bump oxc-parser (#4776) Bumps the npm-oxc-validator group in /studio/backend/core/data_recipe/oxc-validator with 1 update: [oxc-parser](https://github.com/oxc-project/oxc/tree/HEAD/napi/parser). Updates `oxc-parser` from 0.121.0 to 0.123.0 - [Release notes](https://github.com/oxc-project/oxc/releases) - [Changelog](https://github.com/oxc-project/oxc/blob/main/napi/parser/CHANGELOG.md) - [Commits](https://github.com/oxc-project/oxc/commits/crates_v0.123.0/napi/parser) --- updated-dependencies: - dependency-name: oxc-parser dependency-version: 0.123.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: npm-oxc-validator ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- studio/backend/core/data_recipe/oxc-validator/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/studio/backend/core/data_recipe/oxc-validator/package.json b/studio/backend/core/data_recipe/oxc-validator/package.json index d1c765a2e1..111ae2b257 100644 --- a/studio/backend/core/data_recipe/oxc-validator/package.json +++ b/studio/backend/core/data_recipe/oxc-validator/package.json @@ -4,7 +4,7 @@ "version": "0.0.1", "type": "module", "dependencies": { - "oxc-parser": "^0.121.0", + "oxc-parser": "^0.123.0", "oxlint": "^1.51.0" } } From 0087515d5cd5ffd0eb469c15c01a7d0aac3a0040 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 8 Apr 2026 03:35:40 -0700 Subject: [PATCH 05/60] build(deps): bump oxc-parser (#4776) Bumps the npm-oxc-validator group in /studio/backend/core/data_recipe/oxc-validator with 1 update: [oxc-parser](https://github.com/oxc-project/oxc/tree/HEAD/napi/parser). Updates `oxc-parser` from 0.121.0 to 0.123.0 - [Release notes](https://github.com/oxc-project/oxc/releases) - [Changelog](https://github.com/oxc-project/oxc/blob/main/napi/parser/CHANGELOG.md) - [Commits](https://github.com/oxc-project/oxc/commits/crates_v0.123.0/napi/parser) --- updated-dependencies: - dependency-name: oxc-parser dependency-version: 0.123.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: npm-oxc-validator ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> From c3d2d58046ebf5d706d038173ddf661436c4e861 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 8 Apr 2026 03:39:50 -0700 Subject: [PATCH 06/60] Update dependabot.yml (#4915) --- .github/dependabot.yml | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index a06cb1d114..4a0bfa70f1 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -9,19 +9,6 @@ updates: actions: patterns: ["*"] - - package-ecosystem: "pip" - directories: - - "/" - - "/studio/backend/plugins/data-designer-unstructured-seed" - - "/studio/backend/requirements" - - "/unsloth/kernels/moe" - schedule: - interval: "weekly" - open-pull-requests-limit: 10 - groups: - pip: - patterns: ["*"] - - package-ecosystem: "bun" directory: "/studio/frontend" schedule: From 8e977445d43350e3bc74b67544a5fbdbde29bb52 Mon Sep 17 00:00:00 2001 From: Wasim Yousef Said Date: Wed, 8 Apr 2026 12:48:22 +0200 Subject: [PATCH 07/60] Let recipes use the model loaded in Chat (#4840) * feat: inject local model provider into recipe jobs via JWT * feat: auto-generate JWT for local model providers in recipes * feat: add is_local flag to model provider config types and utils * fix(studio): skip endpoint validation for local providers * feat(studio): add local/external model source toggle to provider dialog * feat(studio): thread localProviderNames through model config dialog chain * feat(studio): show 'Local model (Chat)' label for local model_provider configs * fix: hardcode loopback for local endpoint, clear stale creds on toggle * fix: document TOCTOU/JWT rotation, add deferred import comments, fix is_local serialization * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix(studio): clear stale local model state on provider toggle and validation * fix(studio): override empty local endpoint in validation and skip model gate for unused providers * fix(studio): resolve loopback port from app.state, clear stale local provider fields, sync model id on toggle Address review feedback on the local-model-provider flow: - Backend (jobs.py): _resolve_local_v1_endpoint now reads the actual bound port from app.state.server_port (set in run.py after binding) instead of parsing it out of request.base_url, which is wrong behind any reverse proxy or non-default port. The two duplicated urlparse blocks are gone. - Backend (jobs.py): defensively pop api_key_env, extra_headers, extra_body from local providers so a previously external provider that flipped to local cannot leak invalid JSON or rogue auth headers into the local /v1 call. Also dedupe the post-loop assignment and tighten the local-name intersection so empty names cannot match. - Backend (jobs.py): hoist datetime and urllib.parse imports to the top import block for consistency with the rest of the file. - Backend (run.py): expose the bound port on app.state.server_port after the uvicorn server is constructed. - Frontend (model-provider-dialog.tsx): clear extra_headers and extra_body when toggling to local mode. Hidden inputs would otherwise keep stale JSON blocking validate/run. - Frontend (model-config-dialog.tsx): factor the local-aware provider selection logic into applyProviderChange and call it from both onValueChange and onBlur, so manually typing a provider name and tabbing away keeps the model field consistent. - Frontend (recipe-studio.ts store): handle both directions of the is_local toggle in the cascade. external -> local now backfills model: "local" on already-linked model_configs so they pass validation immediately, mirroring the existing local -> external clear path. - Frontend (validate.ts + build-payload.ts): thread localProviderNames into validateModelConfigProviders and skip the "model is required" check for local-linked configs. Local providers do not need a real model id since the inference endpoint uses the loaded Chat model. * fix(studio): narrow store cascade types, sync model placeholder on graph relink and node removal, harden ephemeral port path Loop 2 review fixes: - recipe-studio.ts: type-narrow next.is_local by also checking next.kind === "model_provider". TS otherwise raised TS2339 because next was typed as the union NodeConfig after the spread. The behavior is unchanged but the code now compiles cleanly. - model-config-dialog.tsx: convert the lastProviderRef / providerInputRef ref-during-render pattern (pre-existing react-hooks/refs lint error) to a useEffect that syncs providerInputRef from config.provider. The combobox blur path still uses applyProviderChange and remains stable. - recipe-graph-connection.ts: when a graph drag links a model_provider to a model_config, mirror the dialog applyProviderChange behavior: fill model: "local" if the new provider is local and the model field is blank, clear model when relinking from a local placeholder to an external provider, otherwise leave the model alone. - reference-sync.ts: when a referenced provider node is removed, clear the synthetic model: "local" placeholder along with the provider field, so a future relink to an external provider does not pass validation with a stale value that fails at runtime. - run.py: only publish app.state.server_port when the bound port is a real positive integer; for ephemeral binds (port==0) leave it unset and let request handlers fall back to request.base_url. - jobs.py: _resolve_local_v1_endpoint also falls back when app.state.server_port is non-positive, and uses `is None` instead of the truthy fallback so a literal 0 is handled correctly. * fix(studio): strict is_local check, narrow loaded-model gate to LLM-reachable configs, add scope-server port fallback Loop 3 review fixes: - jobs.py, validate.py: require `is_local is True` instead of truthy check. Malformed payloads such as is_local: "false" or is_local: 1 would otherwise be treated as local and silently rewritten to the loopback endpoint. - jobs.py: _resolve_local_v1_endpoint now tries request.scope["server"] (the actual uvicorn-assigned (host, port) tuple) as a second resolution step before falling back to parsing request.base_url. This covers direct-uvicorn startup paths and ephemeral binds that never publish app.state.server_port. - jobs.py: new _used_llm_model_aliases helper collects the set of model_aliases that an LLM column actually references, and the "Chat model loaded" gate is now only triggered when a local provider is reachable from that set. Orphan model_config nodes on the canvas no longer block unrelated recipe runs. * fix(studio): force skip_health_check on local-linked configs, skip JSON parsing for local providers, local-aware inline editor Loop 4 review fixes: - jobs.py: after rewriting local providers, also force skip_health_check: true on any model_config linked to a local provider. The /v1/models endpoint only advertises the real loaded model id, so data_designer's default model-availability health check would otherwise fail against the placeholder "local" id before the first chat completion call. The inference route already ignores the model id in chat completions, so skipping the check is safe. - builders-model.ts: buildModelProvider now short-circuits for local providers and emits only { name, endpoint: "", provider_type, is_local } without running parseJsonObject on the hidden extra_headers/extra_body inputs. Imported or hydrated recipes with stale invalid JSON in those fields no longer block client-side validate/run. - inline-model.tsx: the model_config branch now accepts an optional localProviderNames prop and mirrors the dialog applyProviderChange behavior. Changing provider to/from a local one auto-fills or clears the "local" placeholder consistently with the other edit paths. - recipe-graph-node.tsx: derive localProviderNames from the store via useMemo (stable identity) and pass it through renderNodeBody to . Hooks order is preserved by declaring them above the early return for markdown_note nodes. - run.py: minor comment tweak - loop 3 already added the scope-server fallback path, note that in the comment. --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: danielhanchen --- studio/backend/routes/data_recipe/jobs.py | 164 ++++++++++++- studio/backend/routes/data_recipe/validate.py | 16 ++ studio/backend/run.py | 8 + .../recipe-studio/blocks/render-dialog.tsx | 2 + .../components/inline/inline-model.tsx | 39 +++- .../components/recipe-graph-node.tsx | 28 ++- .../recipe-studio/dialogs/config-dialog.tsx | 3 + .../dialogs/models/model-config-dialog.tsx | 38 ++- .../dialogs/models/model-provider-dialog.tsx | 218 ++++++++++++------ .../recipe-studio/recipe-studio-page.tsx | 1 + .../stores/helpers/reference-sync.ts | 10 +- .../recipe-studio/stores/recipe-studio.ts | 28 +++ .../src/features/recipe-studio/types/index.ts | 2 + .../recipe-studio/utils/config-factories.ts | 2 + .../utils/graph/recipe-graph-connection.ts | 12 +- .../utils/import/parsers/model-parser.ts | 2 + .../utils/payload/build-payload.ts | 5 + .../utils/payload/builders-model.ts | 20 ++ .../recipe-studio/utils/payload/validate.ts | 9 +- .../recipe-studio/utils/recipe-studio-view.ts | 6 + 20 files changed, 518 insertions(+), 95 deletions(-) diff --git a/studio/backend/routes/data_recipe/jobs.py b/studio/backend/routes/data_recipe/jobs.py index 1d5eceee03..00546b47a4 100644 --- a/studio/backend/routes/data_recipe/jobs.py +++ b/studio/backend/routes/data_recipe/jobs.py @@ -5,7 +5,9 @@ from __future__ import annotations +from datetime import timedelta from typing import Any +from urllib.parse import urlparse from fastapi import APIRouter, HTTPException, Query, Request from fastapi.responses import JSONResponse, StreamingResponse @@ -26,6 +28,161 @@ from models.data_recipe import ( router = APIRouter() +def _resolve_local_v1_endpoint(request: Request) -> str: + """Return the loopback /v1 URL for the actual backend listen port. + + Resolution order: + 1. ``app.state.server_port`` - explicitly published by run.py after + the uvicorn server has bound. This is the most reliable source + because it survives reverse proxies, TLS terminators and tunnels. + 2. ``request.scope["server"]`` - the real (host, port) tuple uvicorn + sets when the request is dispatched. Used when Studio is started + outside ``run_server`` (e.g. ``uvicorn studio.backend.main:app``). + 3. ``request.base_url`` parsed - last resort for test fixtures that + do not route through a live uvicorn server. + """ + port: Any = getattr(request.app.state, "server_port", None) + if not isinstance(port, int) or port <= 0: + server = request.scope.get("server") + if ( + isinstance(server, tuple) + and len(server) >= 2 + and isinstance(server[1], int) + and server[1] > 0 + ): + port = server[1] + else: + parsed = urlparse(str(request.base_url)) + port = parsed.port if parsed.port is not None else 8888 + return f"http://127.0.0.1:{int(port)}/v1" + + +def _used_llm_model_aliases(recipe: dict[str, Any]) -> set[str]: + """Return the set of model_aliases that are actually referenced by an + LLM column. Used to narrow the "Chat model loaded" gate so that orphan + model_config nodes on the canvas do not block unrelated recipe runs. + + The ``llm-`` prefix matches the existing convention in + ``core/data_recipe/service.py::_recipe_has_llm_columns`` and covers all + LLM column types emitted by the frontend (llm-text, llm-code, + llm-structured, llm-judge). + """ + aliases: set[str] = set() + for column in recipe.get("columns", []): + if not isinstance(column, dict): + continue + column_type = column.get("column_type") + if not isinstance(column_type, str) or not column_type.startswith("llm-"): + continue + alias = column.get("model_alias") + if isinstance(alias, str) and alias: + aliases.add(alias) + return aliases + + +def _inject_local_providers(recipe: dict[str, Any], request: Request) -> None: + """ + Mutate recipe dict in-place: for any provider with is_local=True, + generate a JWT and fill in the endpoint pointing at this server. + """ + providers = recipe.get("model_providers") + if not providers: + return + + # Collect local providers and pop is_local from ALL dicts unconditionally. + # Strict `is True` guard so malformed payloads (is_local: 1, + # is_local: "true") do not accidentally trigger the loopback rewrite. + local_indices: list[int] = [] + for i, provider in enumerate(providers): + if not isinstance(provider, dict): + continue + is_local = provider.pop("is_local", None) + if is_local is True: + local_indices.append(i) + + if not local_indices: + return + + endpoint = _resolve_local_v1_endpoint(request) + + # Only gate on model-loaded if a local provider is actually reachable + # from an LLM column through a model_config. Orphan model_config nodes + # that reference a local provider but that no LLM column uses should + # not block runs; the recipe would never call /v1 for them. + local_names = { + providers[i].get("name") for i in local_indices if providers[i].get("name") + } + used_aliases = _used_llm_model_aliases(recipe) + referenced_providers = { + mc.get("provider") + for mc in recipe.get("model_configs", []) + if ( + isinstance(mc, dict) + and mc.get("provider") + and mc.get("alias") in used_aliases + ) + } + + token = "" + if local_names & referenced_providers: + # Verify a model is loaded. + # NOTE: This is a point-in-time check (TOCTOU). The model could be unloaded + # or swapped after this check but before the recipe subprocess calls /v1. + # The inference endpoint returns a clear 400 in that case. + # + # Imports are deferred to avoid circular dependencies with inference modules. + from routes.inference import get_llama_cpp_backend + from core.inference import get_inference_backend + + llama = get_llama_cpp_backend() + model_loaded = llama.is_loaded + if not model_loaded: + backend = get_inference_backend() + model_loaded = bool(backend.active_model_name) + if not model_loaded: + raise ValueError( + "No model loaded in Chat. Load a model first, then run the recipe." + ) + + from auth.authentication import ( + create_access_token, + ) # deferred: avoids circular import + + # Uses the "unsloth" admin subject. If the user changes their password, + # the JWT secret rotates and this token becomes invalid mid-run. + # Acceptable for v1 - recipes typically finish well within one session. + token = create_access_token( + subject = "unsloth", + expires_delta = timedelta(hours = 24), + ) + + # Defensively strip any stale "external"-only fields the frontend may + # have left on the dict (extra_headers/extra_body/api_key_env). The UI + # hides these inputs in local mode but the payload builder still serializes + # them, so a previously external provider that flipped to local can carry + # invalid JSON or rogue auth headers into the local /v1 call. + for i in local_indices: + providers[i]["endpoint"] = endpoint + providers[i]["api_key"] = token + providers[i]["provider_type"] = "openai" + providers[i].pop("api_key_env", None) + providers[i].pop("extra_headers", None) + providers[i].pop("extra_body", None) + + # Force skip_health_check on any model_config that references a local + # provider. The local /v1/models endpoint only lists the real loaded + # model (e.g. "unsloth/llama-3.2-1b") and not the placeholder "local" + # that the recipe sends as the model id, so data_designer's pre-flight + # health check would otherwise fail before the first completion call. + # The backend route ignores the model id field in chat completions, so + # skipping the check is safe. + for mc in recipe.get("model_configs", []): + if not isinstance(mc, dict): + continue + if mc.get("provider") in local_names: + mc["skip_health_check"] = True + + def _normalize_run_name(value: Any) -> str | None: if value is None: return None @@ -40,7 +197,7 @@ def _normalize_run_name(value: Any) -> str | None: @router.post("/jobs", response_class = JSONResponse, response_model = JobCreateResponse) -def create_job(payload: RecipePayload): +def create_job(payload: RecipePayload, request: Request): recipe = payload.recipe if not recipe.get("columns"): raise HTTPException(status_code = 400, detail = "Recipe must include columns.") @@ -67,6 +224,11 @@ def create_job(payload: RecipePayload): status_code = 400, detail = f"invalid run_config: {exc}" ) from exc + try: + _inject_local_providers(recipe, request) + except ValueError as exc: + raise HTTPException(status_code = 400, detail = str(exc)) from exc + mgr = get_job_manager() try: job_id = mgr.start(recipe = recipe, run = run) diff --git a/studio/backend/routes/data_recipe/validate.py b/studio/backend/routes/data_recipe/validate.py index a793a3b172..555e3eaa06 100644 --- a/studio/backend/routes/data_recipe/validate.py +++ b/studio/backend/routes/data_recipe/validate.py @@ -68,6 +68,20 @@ def _collect_validation_errors(recipe: dict[str, Any]) -> list[ValidateError]: return errors +def _patch_local_providers(recipe: dict[str, Any]) -> None: + """Strip is_local and fill a dummy endpoint so validation doesn't choke. + + Uses a strict `is True` check to match _inject_local_providers in + jobs.py - malformed payloads with truthy but non-boolean is_local + values should not be treated as local. + """ + for provider in recipe.get("model_providers", []): + if not isinstance(provider, dict): + continue + if provider.pop("is_local", None) is True: + provider["endpoint"] = "http://127.0.0.1" + + @router.post("/validate", response_model = ValidateResponse) def validate(payload: RecipePayload) -> ValidateResponse: recipe = payload.recipe @@ -77,6 +91,8 @@ def validate(payload: RecipePayload) -> ValidateResponse: errors = [ValidateError(message = "Recipe must include columns.")], ) + _patch_local_providers(recipe) + try: validate_recipe(recipe) except RuntimeError as exc: diff --git a/studio/backend/run.py b/studio/backend/run.py index 9c3622988e..86c1194661 100644 --- a/studio/backend/run.py +++ b/studio/backend/run.py @@ -324,6 +324,14 @@ def run_server( _server = uvicorn.Server(config) _shutdown_event = Event() + # Expose the actual bound port so request-handling code can build + # loopback URLs that point at the real backend, not whatever port a + # reverse proxy or tunnel exposed in the request URL. Only publish + # an explicit value when we know the concrete port; for ephemeral + # binds (port==0) leave it unset and let request handlers fall back + # to the ASGI request scope or request.base_url. + app.state.server_port = port if port and port > 0 else None + # Run server in a daemon thread def _run(): asyncio.run(_server.serve()) diff --git a/studio/frontend/src/features/recipe-studio/blocks/render-dialog.tsx b/studio/frontend/src/features/recipe-studio/blocks/render-dialog.tsx index 10fcaa489d..92f72dfff1 100644 --- a/studio/frontend/src/features/recipe-studio/blocks/render-dialog.tsx +++ b/studio/frontend/src/features/recipe-studio/blocks/render-dialog.tsx @@ -28,6 +28,7 @@ export function renderBlockDialog( categoryOptions: SamplerConfig[], modelConfigAliases: string[], modelProviderOptions: string[], + localProviderNames: Set, toolProfileAliases: string[], datetimeOptions: string[], onUpdate: (id: string, patch: Partial) => void, @@ -109,6 +110,7 @@ export function renderBlockDialog( ) : null; diff --git a/studio/frontend/src/features/recipe-studio/components/inline/inline-model.tsx b/studio/frontend/src/features/recipe-studio/components/inline/inline-model.tsx index d3ca22b00b..16e99f4fae 100644 --- a/studio/frontend/src/features/recipe-studio/components/inline/inline-model.tsx +++ b/studio/frontend/src/features/recipe-studio/components/inline/inline-model.tsx @@ -10,11 +10,21 @@ type InlineModelPatch = Partial | Partial; type InlineModelProps = { config: ModelProviderConfig | ModelConfig; + localProviderNames?: Set; onUpdate: (patch: InlineModelPatch) => void; }; export function InlineModel(props: InlineModelProps): ReactElement { if (props.config.kind === "model_provider") { + if (props.config.is_local) { + return ( +
+ + Local model (Chat) + +
+ ); + } return (
@@ -42,21 +52,40 @@ export function InlineModel(props: InlineModelProps): ReactElement { ); } + // model_config branch - mirror the local-aware provider sync from the + // dialog path so inline edits do not leave stale "local" placeholders + // on external providers and fill the placeholder when switching to local. + const localNames = props.localProviderNames ?? new Set(); + const modelConfig = props.config; + const handleProviderChange = (nextProvider: string) => { + const isLocal = localNames.has(nextProvider); + if (isLocal && !modelConfig.model.trim()) { + props.onUpdate({ provider: nextProvider, model: "local" }); + return; + } + if (!isLocal && modelConfig.model === "local") { + props.onUpdate({ provider: nextProvider, model: "" }); + return; + } + props.onUpdate({ provider: nextProvider }); + }; + const isLinkedToLocal = localNames.has(modelConfig.provider); + return (
props.onUpdate({ provider: event.target.value })} + value={modelConfig.provider} + onChange={(event) => handleProviderChange(event.target.value)} /> props.onUpdate({ model: event.target.value })} /> @@ -65,7 +94,7 @@ export function InlineModel(props: InlineModelProps): ReactElement { className="nodrag h-8 w-full text-xs" type="number" placeholder="0.7" - value={props.config.inference_temperature ?? ""} + value={modelConfig.inference_temperature ?? ""} onChange={(event) => props.onUpdate({ // biome-ignore lint/style/useNamingConvention: api schema diff --git a/studio/frontend/src/features/recipe-studio/components/recipe-graph-node.tsx b/studio/frontend/src/features/recipe-studio/components/recipe-graph-node.tsx index 0fcf202190..8afc4e26bf 100644 --- a/studio/frontend/src/features/recipe-studio/components/recipe-graph-node.tsx +++ b/studio/frontend/src/features/recipe-studio/components/recipe-graph-node.tsx @@ -30,7 +30,7 @@ import { Position, useUpdateNodeInternals, } from "@xyflow/react"; -import { type ReactElement, memo, useEffect } from "react"; +import { type ReactElement, memo, useEffect, useMemo } from "react"; import { MAX_NODE_WIDTH, MAX_NOTE_NODE_WIDTH, @@ -287,6 +287,7 @@ function renderNodeBody( config: NodeConfig | undefined, summary: string, updateConfig: (id: string, patch: Partial) => void, + localProviderNames: Set, ): ReactElement { if (config?.kind === "markdown_note") { return ; @@ -300,7 +301,13 @@ function renderNodeBody( return ; } if (config.kind === "model_provider" || config.kind === "model_config") { - return ; + return ( + + ); } if (config.kind === "llm") { return ; @@ -355,6 +362,16 @@ function RecipeGraphNodeBase({ const config = useRecipeStudioStore((state) => state.configs[id]); const openConfig = useRecipeStudioStore((state) => state.openConfig); const updateConfig = useRecipeStudioStore((state) => state.updateConfig); + const allConfigs = useRecipeStudioStore((state) => state.configs); + const localProviderNames = useMemo(() => { + const names = new Set(); + for (const cfg of Object.values(allConfigs)) { + if (cfg.kind === "model_provider" && cfg.is_local === true) { + names.add(cfg.name); + } + } + return names; + }, [allConfigs]); const llmAuxVisible = useRecipeStudioStore( (state) => state.llmAuxVisibility[id] ?? false, ); @@ -418,7 +435,12 @@ function RecipeGraphNodeBase({ data.kind === "tool_config" || data.kind === "validator"; const summary = getConfigSummary(config); - const nodeBody = renderNodeBody(config, summary, updateConfig); + const nodeBody = renderNodeBody( + config, + summary, + updateConfig, + localProviderNames, + ); const canShowLlmAux = config?.kind === "llm" && (Boolean(config.prompt.trim()) || diff --git a/studio/frontend/src/features/recipe-studio/dialogs/config-dialog.tsx b/studio/frontend/src/features/recipe-studio/dialogs/config-dialog.tsx index 6457993c35..c62169956d 100644 --- a/studio/frontend/src/features/recipe-studio/dialogs/config-dialog.tsx +++ b/studio/frontend/src/features/recipe-studio/dialogs/config-dialog.tsx @@ -18,6 +18,7 @@ type ConfigDialogProps = { categoryOptions: SamplerConfig[]; modelConfigAliases: string[]; modelProviderOptions: string[]; + localProviderNames: Set; toolProfileAliases: string[]; datetimeOptions: string[]; onUpdate: (id: string, patch: Partial) => void; @@ -32,6 +33,7 @@ export function ConfigDialog({ categoryOptions, modelConfigAliases, modelProviderOptions, + localProviderNames, toolProfileAliases, datetimeOptions, onUpdate, @@ -101,6 +103,7 @@ export function ConfigDialog({ categoryOptions, modelConfigAliases, modelProviderOptions, + localProviderNames, toolProfileAliases, datetimeOptions, onUpdate, diff --git a/studio/frontend/src/features/recipe-studio/dialogs/models/model-config-dialog.tsx b/studio/frontend/src/features/recipe-studio/dialogs/models/model-config-dialog.tsx index 3192dc8fd9..368ae08acb 100644 --- a/studio/frontend/src/features/recipe-studio/dialogs/models/model-config-dialog.tsx +++ b/studio/frontend/src/features/recipe-studio/dialogs/models/model-config-dialog.tsx @@ -17,7 +17,7 @@ import { } from "@/components/ui/combobox"; import { Input } from "@/components/ui/input"; import { Textarea } from "@/components/ui/textarea"; -import { type ReactElement, useRef, useState } from "react"; +import { type ReactElement, useEffect, useRef, useState } from "react"; import type { ModelConfig } from "../../types"; import { CollapsibleSectionTriggerButton } from "../shared/collapsible-section-trigger"; import { FieldLabel } from "../shared/field-label"; @@ -26,14 +26,17 @@ import { NameField } from "../shared/name-field"; type ModelConfigDialogProps = { config: ModelConfig; providerOptions: string[]; + localProviderNames: Set; onUpdate: (patch: Partial) => void; }; export function ModelConfigDialog({ config, providerOptions, + localProviderNames, onUpdate, }: ModelConfigDialogProps): ReactElement { + const isLinkedToLocal = localProviderNames.has(config.provider); const [optionalOpen, setOptionalOpen] = useState(false); const modelId = `${config.id}-model`; const providerId = `${config.id}-provider`; @@ -44,11 +47,13 @@ export function ModelConfigDialog({ const extraBodyId = `${config.id}-inference-extra-body`; const providerAnchorRef = useRef(null); const providerInputRef = useRef(config.provider); - const lastProviderRef = useRef(config.provider); - if (lastProviderRef.current !== config.provider) { - lastProviderRef.current = config.provider; + // Sync providerInputRef with the current provider value. Updating a ref in + // an effect (vs reading/writing it during render) satisfies the + // react-hooks/refs rule and keeps the combobox blur path stable across + // re-renders. + useEffect(() => { providerInputRef.current = config.provider; - } + }, [config.provider]); const updateField = ( key: K, value: ModelConfig[K], @@ -56,6 +61,21 @@ export function ModelConfigDialog({ onUpdate({ [key]: value } as Partial); }; + // Apply provider selection while keeping the local-provider model autofill + // consistent across both dropdown selection and free-typed + blur input. + const applyProviderChange = (selectedProvider: string) => { + const isLocal = localProviderNames.has(selectedProvider); + if (isLocal && !config.model.trim()) { + onUpdate({ provider: selectedProvider, model: "local" }); + return; + } + if (!isLocal && config.model === "local") { + onUpdate({ provider: selectedProvider, model: "" }); + return; + } + updateField("provider", selectedProvider); + }; + return (
updateField("provider", value ?? "")} + onValueChange={(value) => applyProviderChange(value ?? "")} onInputValueChange={(value) => { providerInputRef.current = value; }} @@ -98,7 +118,7 @@ export function ModelConfigDialog({ onBlur={() => { const next = providerInputRef.current; if (next !== config.provider) { - updateField("provider", next); + applyProviderChange(next); } }} /> @@ -124,12 +144,12 @@ export function ModelConfigDialog({ updateField("model", event.target.value)} /> diff --git a/studio/frontend/src/features/recipe-studio/dialogs/models/model-provider-dialog.tsx b/studio/frontend/src/features/recipe-studio/dialogs/models/model-provider-dialog.tsx index 897fc7ee36..ef7366a225 100644 --- a/studio/frontend/src/features/recipe-studio/dialogs/models/model-provider-dialog.tsx +++ b/studio/frontend/src/features/recipe-studio/dialogs/models/model-provider-dialog.tsx @@ -24,6 +24,7 @@ export function ModelProviderDialog({ onUpdate, }: ModelProviderDialogProps): ReactElement { const [optionalOpen, setOptionalOpen] = useState(false); + const isLocal = config.is_local ?? false; const endpointId = `${config.id}-endpoint`; const apiKeyEnvId = `${config.id}-api-key-env`; const apiKeyId = `${config.id}-api-key`; @@ -43,94 +44,163 @@ export function ModelProviderDialog({ value={config.name} onChange={(value) => onUpdate({ name: value })} /> -
-

- Start with the endpoint you want this model to use -

-

- Most connections only need an endpoint. Add an API key if that - service requires one. -

-
+ + {/* Model source toggle */}
- - updateField("endpoint", event.target.value)} - /> +

Model source

+
+ + +
-
- - updateField("api_key", event.target.value)} - /> -
- - - - - + + {isLocal ? ( +
+

+ Ready to go +

+

+ Recipes will use whatever model is loaded in the Chat tab when you + hit run. No endpoint or API key needed. +

+
+ ) : ( + <> +
+

+ Start with the endpoint you want this model to use +

+

+ Most connections only need an endpoint. Add an API key if that + service requires one. +

+
updateField("api_key_env", event.target.value)} + placeholder="https://..." + value={config.endpoint} + onChange={(event) => updateField("endpoint", event.target.value)} />
-