From 1dd2fc45837c9ae2939879df4ca5629fbf7b7eab Mon Sep 17 00:00:00 2001 From: Leo Borcherding Date: Mon, 27 Jul 2026 01:31:56 -0500 Subject: [PATCH] tests: read checked-in files as UTF-8 instead of the platform default (#7438) * tests: read checked-in files as UTF-8 instead of the platform default Path.read_text() with no encoding uses locale.getpreferredencoding(), which is UTF-8 on the Linux runners and cp1252 on a stock Windows install. Nine module-level reads of checked-in source files were relying on that default. studio/backend/routes/inference.py carries the DeepSeek tool-call token regexes, so it holds U+FF5C and U+2581. Under cp1252 that read raised UnicodeDecodeError on byte 0x81 at position 97806, and because the reads run at import time it took test_cancel_atomicity.py and test_cancel_id_wiring.py out at collection, not as failures. Green on CI, permanently broken for a Windows contributor running the suite locally. Adds a guard: at module scope there is no tmp_path fixture, so a bare read_text()/write_text()/open() there is always touching a checked-in file. That makes the rule mechanical enough to enforce with no allowlist, while staying quiet about temp-dir I/O inside test bodies where the platform default is harmless. The repo already spells this correctly in 464 other places; this only stops the stragglers coming back. * tests: cover import-time helper reads and keep the guard py3.9-safe Follows up on the Codex review: - add `from __future__ import annotations`, since `str | None` in `_offender` is evaluated at import on Python 3.9 and pyproject declares requires-python ">=3.9,<3.15". - widen the guard from module scope to import time. Class bodies and the bodies of module-level helpers called from an executing statement run during collection too, so `CODE = _extract_mixed_precision_code()` was the same hazard as an inline read. `if __name__ == "__main__":` blocks are skipped: pytest never executes them. - scan studio/backend/tests/ as well as tests/. Both trees are collected on Windows by separate CI jobs, and the offender that started this, test_tool_xml_strip.py reading routes/inference.py, lives there. Widening it surfaced seven more import-time reads of checked-in sources; all now name utf-8. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Harden the import-time encoding guard for PR #7438 Close the detector gaps raised in review, all of which I reproduced against the actual AST before changing anything. False negatives (the guard let a real hazard through): - _is_main_guard ignored the comparison operator, so if __name__ != "__main__" counted as script-only even though its body runs at import. - The else arm of a main guard was discarded with the rest of the If node. - Decorators and argument defaults on a module-level def were skipped with the body, though both are evaluated when the def executes. - Path.open() in text mode was invisible; only builtin open() was matched. - encoding = None and encoding = "locale" both re-select the platform default, but the keyword merely being present counted as pinned. False positives (the guard would have blocked a compliant contributor): - A non-literal mode fell through to the "r" default, so open(p, mode) was flagged even when mode is "rb", where adding encoding= is a ValueError and there is no edit that satisfies the rule. - Same for open(*args) and a **kwargs splat, which hide the mode and can hide an encoding. - Lambda bodies and comprehension elements were walked even though neither runs at definition. Verified: still reports the same 22 offenders on unpatched main, green on this branch and on the tree merged with latest main (557 files), and an adversarial corpus of 33 cases now scores zero false positives and zero false negatives. Also corrected two docstring claims: neither collecting job runs on Windows, and the read is governed by locale.getencoding(). * Walk eager comprehensions and treat io.open as the builtin Two regressions from the previous commit, both reproduced against the AST before changing anything. Lumping list, set and dict comprehensions in with generator expressions was wrong. Only a genexp is lazy; the other three run their element expression, their filters and their nested iterators immediately, so CONTENTS = [p.read_text() for p in PATHS] at module scope is an import-time read the guard was silently missing. Comprehensions are now walked in full and only the genexp keeps the outermost-iterable-only treatment. io was also in the not-a-path-opener list, but io.open is the builtin, with the same mode position and the same platform default. io.open(CHECKED_IN_FILE) is exactly the hazard this guard exists for, so it is matched now, with binary modes and a pinned encoding still exempt. tarfile.open and fitz.open stay exempt since neither has an encoding to name. Verified: 13 targeted cases covering all five eager comprehension forms and io.open in text, binary and pinned shapes all classify correctly; still 22 offenders on unpatched main; green on this branch and on the tree merged with latest main. * Close three more walker gaps in the import-time guard All three reproduced against the AST first. A generator expression handed straight to a call is consumed there, so DATA = "".join(p.read_text() for p in paths) runs its element at import. Only an unconsumed genexp bound to a name stays lazy, so the walker now follows the consumed ones in full and keeps the outermost-iterable-only treatment for the rest. if "__main__" == __name__ is an equivalent and accepted spelling of the main guard, but requiring __name__ on the left meant its body was treated as import-time code. That is a false positive on a block pytest never runs, so both operand orders are recognised now. The helper table was built from module-level defs only, so a def in a class body invoked while the class is constructed was never followed, contradicting the walker's stated coverage of class bodies. Helpers are now collected from the module body and from class bodies at any nesting. Verified: 15 targeted cases including all three fixes and the earlier ones still classify correctly; still 22 offenders on unpatched main; green on this branch and on the tree merged with latest main. * Handle positional read_text encodings, lazy generators and nested helpers * Guard reads reached from test bodies, unbound Path calls and __file__ paths * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Follow derived paths, skip lazy generator helpers, cover compressed openers * Guard the CLI tests, helper parameters and unbound Path arguments * Discover test roots and follow literal, in-place and tuple-derived paths * Identify module openers by import, unwrap starred paths, pin subprocess snippets * Resolve import origins, seed helper locals, follow named generators and parametrize * Scope imports lexically, list tracked test files, bind unpacked names * Resolve aliased openers, keyword-only params, destructured targets, next() * Pin the encoding on subprocess snippets, workflow lint and CLI output for PR #7438 * Harden the CLI encoding guard against detached streams for PR #7438 * Tighten the encoding guard's path and scope analysis for PR #7438 * Resolve path provenance more precisely and keep POSIX stream encodings for PR #7438 * Resolve qualified path classes and scope conditional imports for PR #7438 * Scope CLI stream setup to the entry point and align two encoding pairs for PR #7438 --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: danielhanchen --- scripts/lint_workflow_triggers.py | 6 +- .../backend/tests/test_cloudflare_tunnel.py | 10 +- studio/backend/tests/test_consent_gate.py | 20 +- studio/backend/tests/test_cpu_threads.py | 2 +- studio/backend/tests/test_data_recipe_seed.py | 2 +- studio/backend/tests/test_desktop_auth.py | 8 +- .../tests/test_gguf_load_cache_reuse.py | 6 +- studio/backend/tests/test_host_defaults.py | 4 +- studio/backend/tests/test_mcp_servers.py | 4 +- .../tests/test_mlx_training_worker_config.py | 8 +- studio/backend/tests/test_mtp_vram_budget.py | 36 +- .../tests/test_native_context_length.py | 2 +- .../test_native_template_trust_remote_code.py | 6 +- .../tests/test_offline_inference_parent.py | 2 +- .../test_recommended_folders_has_model.py | 2 +- .../test_recommended_folders_permission.py | 2 +- studio/backend/tests/test_sandbox_tools.py | 8 +- .../tests/test_security_gate_consistency.py | 10 +- studio/backend/tests/test_ssm_runtime.py | 14 +- studio/backend/tests/test_studio_api.py | 27 +- .../tests/test_tool_call_parser_strict.py | 16 +- studio/backend/tests/test_tool_xml_strip.py | 2 +- .../tests/test_tp_vision_regression.py | 8 +- .../tests/test_training_raw_support.py | 8 +- .../tests/test_transformers_version.py | 2 +- .../test_yaml_trust_remote_code_removed.py | 6 +- .../test_cpo_processor_text_tokenizer.py | 4 +- .../test_dpo_vision_processor_passthrough.py | 2 +- tests/python/test_e2e_no_torch_sandbox.py | 30 +- .../test_fast_language_model_text_only.py | 2 +- .../test_fast_model_config_passthrough.py | 2 +- tests/python/test_gpu_init_ldconfig_guard.py | 6 +- tests/python/test_grpo_ddp_model_config.py | 2 +- .../test_orpo_processor_text_tokenizer.py | 2 +- tests/python/test_pad_token_fix.py | 2 +- tests/python/test_studio_import_no_torch.py | 18 +- tests/python/test_v100_fullft_precision.py | 2 +- tests/python/test_vision_lora_targeting.py | 2 +- .../test_fix_sentencepiece_gguf_robustness.py | 4 +- tests/security/test_scan_packages.py | 10 +- .../install/test_llama_pr_force_and_source.py | 4 +- .../install/test_managed_node_runtime.py | 6 +- tests/studio/install/test_pr4562_bugfixes.py | 46 +- .../load_freeze/test_load_orchestrator.py | 6 +- tests/studio/playwright_chat_ime_i18n.py | 10 +- tests/studio/playwright_chat_ui.py | 2 +- tests/studio/studio_api_smoke.py | 2 +- tests/studio/test_auth_form_input_count.py | 28 +- tests/studio/test_cancel_atomicity.py | 2 +- tests/studio/test_cancel_id_wiring.py | 12 +- .../test_chat_preset_builtin_invariants.py | 8 +- tests/studio/test_chat_prompt_variables.py | 4 +- .../test_chat_response_details_ui_contract.py | 20 +- tests/studio/test_chat_title_generation.py | 14 +- tests/studio/test_cli_run_alias.py | 4 +- tests/studio/test_cli_studio_defaults.py | 13 +- .../test_composer_rtl_bidi_attribute.py | 42 +- .../test_export_output_path_contract.py | 12 +- tests/studio/test_frontend_dep_removal.py | 18 +- tests/studio/test_is_mlx_dispatch_gate.py | 2 +- tests/studio/test_llama_cpp_wall_clock_cap.py | 2 +- .../test_mlx_training_worker_behaviors.py | 14 +- tests/studio/test_model_picker_contracts.py | 2 +- .../test_studio_gguf_export_script_pin.py | 2 +- .../test_studio_text_descender_clipping.py | 2 +- tests/test_fast_generate_slow_guard.py | 2 +- tests/test_fp8_device_context.py | 4 +- tests/test_gemma4_chat_template.py | 2 +- tests/test_gemma_2b_mapper_key.py | 2 +- tests/test_generate_kwarg_gate.py | 2 +- tests/test_gradient_checkpointing_restore.py | 6 +- tests/test_import_fixes_drift.py | 2 +- tests/test_loader_glob_skip.py | 2 +- tests/test_multi_image_grpo_chunking.py | 2 +- tests/test_offload_embedding_hooks.py | 2 +- tests/test_offload_tied_guard.py | 2 +- tests/test_source_read_encoding.py | 1252 +++++++++++++++++ tests/test_studio_install_workspace_guard.py | 89 +- tests/test_studio_root_resilience.py | 4 +- tests/test_tool_mask_zoo_compat.py | 2 +- tests/utils/test_prepare_inputs_leftpad.py | 4 +- tests/utils/test_rope_scaling_drift.py | 6 +- unsloth_cli/__init__.py | 27 +- unsloth_cli/tests/test_start.py | 6 +- 84 files changed, 1652 insertions(+), 352 deletions(-) create mode 100644 tests/test_source_read_encoding.py diff --git a/scripts/lint_workflow_triggers.py b/scripts/lint_workflow_triggers.py index 0688f6c65c..8f22fcaf45 100644 --- a/scripts/lint_workflow_triggers.py +++ b/scripts/lint_workflow_triggers.py @@ -52,14 +52,14 @@ def _normalise_on(on_field): def _load_workflow(path: Path): try: - return yaml.safe_load(path.read_text()) + return yaml.safe_load(path.read_text(encoding = "utf-8")) except Exception as exc: print(f"ERROR: failed to parse {path}: {exc}", file = sys.stderr) sys.exit(2) def _extract_cache_keys(path: Path) -> list[str]: - text = path.read_text() + text = path.read_text(encoding = "utf-8") keys: list[str] = [] for m in re.finditer(r"(?:^|\n)\s*key:\s*([^\n]+)", text): keys.append(m.group(1).strip()) @@ -104,7 +104,7 @@ def main() -> int: for t in RESTRICTED_TRIGGERS: if t in triggers: - text = path.read_text() + text = path.read_text(encoding = "utf-8") if "lint:workflow_triggers-allow-workflow_run" not in text: findings.append( f"{path.name}: RESTRICTED trigger '{t}' requires an " diff --git a/studio/backend/tests/test_cloudflare_tunnel.py b/studio/backend/tests/test_cloudflare_tunnel.py index 2094d15066..8d19f09bae 100644 --- a/studio/backend/tests/test_cloudflare_tunnel.py +++ b/studio/backend/tests/test_cloudflare_tunnel.py @@ -915,17 +915,17 @@ def _argparse_default(source, option): def test_run_server_cloudflare_default_off(): - defaults = _func_param_defaults(_RUN_PY.read_text(), "run_server") + defaults = _func_param_defaults(_RUN_PY.read_text(encoding = "utf-8"), "run_server") assert "cloudflare" in defaults assert defaults["cloudflare"] is None def test_argparse_cloudflare_default_off(): - assert _argparse_default(_RUN_PY.read_text(), "--cloudflare") is None + assert _argparse_default(_RUN_PY.read_text(encoding = "utf-8"), "--cloudflare") is None def test_verify_global_reachability_marks_private_address_unreachable(): - src = _RUN_PY.read_text() + src = _RUN_PY.read_text(encoding = "utf-8") tree = ast.parse(src) func_src = next( ast.get_source_segment(src, n) @@ -949,7 +949,7 @@ def test_verify_global_reachability_marks_private_address_unreachable(): def test_run_server_registers_tunnel_atexit_backstop(): # An abnormal exit (exception after startup -> sys.exit) bypasses # _graceful_shutdown; an atexit backstop must still stop the tunnel. - src = _RUN_PY.read_text() + src = _RUN_PY.read_text(encoding = "utf-8") assert "atexit.register(stop_studio_tunnel)" in src @@ -965,7 +965,7 @@ def _run_print_cloudflare_line( color = False, ): """Exec _print_cloudflare_line without importing run.py's heavy deps.""" - src = _RUN_PY.read_text() + src = _RUN_PY.read_text(encoding = "utf-8") tree = ast.parse(src) func_src = next( ast.get_source_segment(src, n) diff --git a/studio/backend/tests/test_consent_gate.py b/studio/backend/tests/test_consent_gate.py index 181e0c9fad..c87662edc1 100644 --- a/studio/backend/tests/test_consent_gate.py +++ b/studio/backend/tests/test_consent_gate.py @@ -402,7 +402,7 @@ class TestWorkersWireTheGate: ], ) def test_worker_invokes_gate(self, rel): - src = (Path(__file__).resolve().parent.parent / rel).read_text() + src = (Path(__file__).resolve().parent.parent / rel).read_text(encoding = "utf-8") assert "evaluate_remote_code_consent" in src assert "remote_code_blocked" in src assert ".blocked" in src @@ -410,14 +410,14 @@ class TestWorkersWireTheGate: def test_mlx_training_path_gates_before_load(self): # The Apple-Silicon path returns before run_training_process's gate, so it must # scan before FastMLXModel.from_pretrained runs repo code. - src = (_BACKEND / "core/training/worker.py").read_text() + src = (_BACKEND / "core/training/worker.py").read_text(encoding = "utf-8") head = src[: src.index("FastMLXModel.from_pretrained(")] assert "evaluate_remote_code_consent" in head def test_lora_base_model_is_gated(self): # Inference + export expand the consent scan to the LoRA base model's code. for rel in ("core/inference/worker.py", "core/export/worker.py"): - src = (_BACKEND / rel).read_text() + src = (_BACKEND / rel).read_text(encoding = "utf-8") assert "evaluate_remote_code_consent" in src assert "get_base_model_from_lora" in src or "mc.base_model" in src @@ -431,12 +431,12 @@ class TestWorkersWireTheGate: "core/training/worker.py", "core/export/worker.py", ): - src = (_BACKEND / rel).read_text() + src = (_BACKEND / rel).read_text(encoding = "utf-8") assert "get_base_model_from_lora_identifier" in src, rel def test_embedding_training_path_gates_before_load(self): # The embedding pipeline must run the malware + consent gates before loading, like the other paths. - src = (_BACKEND / "core/training/worker.py").read_text() + src = (_BACKEND / "core/training/worker.py").read_text(encoding = "utf-8") start = src.index("def _run_embedding_training(") end = src.index("FastSentenceTransformer.from_pretrained(", start) region = src[start:end] @@ -505,7 +505,9 @@ class TestStructuredFindingsForDialog: assert d.findings and d.fingerprint # structured findings for the UI def test_scan_route_uses_preflight(self): - src = (Path(__file__).resolve().parent.parent / "routes/models.py").read_text() + src = (Path(__file__).resolve().parent.parent / "routes/models.py").read_text( + encoding = "utf-8" + ) assert "remote-code-scan" in src # The scan route pins one combined fingerprint over adapter + base, so adapter code is reviewed and approvable too. assert "preflight_remote_code_consent_for_targets" in src @@ -636,7 +638,7 @@ class TestStructuredFindingsForDialog: ], ) def test_fingerprint_threaded_to_worker(self, rel): - src = (Path(__file__).resolve().parent.parent / rel).read_text() + src = (Path(__file__).resolve().parent.parent / rel).read_text(encoding = "utf-8") assert "approved_remote_code_fingerprint" in src # The per-user approval cache rides the same path as the fingerprint. assert "subject" in src @@ -738,7 +740,7 @@ class TestNemotronGateUsesTrustCheck: ], ) def test_worker_nemotron_block_calls_trust_check(self, rel): - src = (_BACKEND / rel).read_text() + src = (_BACKEND / rel).read_text(encoding = "utf-8") assert "_NEMOTRON_TRUST_SUBSTRINGS" in src assert "is_trusted_org_repo(" in src @@ -1525,6 +1527,6 @@ class TestDiscardRemoteCodeDownload: assert res == {"deleted": False, "reason": "not_cached"} def test_route_source_reports_created_by_scan(self): - src = (_BACKEND / "routes/models.py").read_text() + src = (_BACKEND / "routes/models.py").read_text(encoding = "utf-8") assert "created_by_scan" in src assert "discard-remote-code" in src diff --git a/studio/backend/tests/test_cpu_threads.py b/studio/backend/tests/test_cpu_threads.py index 9d8795b6c0..eb3c021ad5 100644 --- a/studio/backend/tests/test_cpu_threads.py +++ b/studio/backend/tests/test_cpu_threads.py @@ -120,7 +120,7 @@ def _ast_line_of_platform_compat_import(source: str) -> int: # run.py and main.py. Robust to formatting / line shifts. @pytest.mark.parametrize("entry_point", [_RUN_PY, _MAIN_PY]) def test_cpu_thread_configuration_runs_before_backend_imports(entry_point): - source = entry_point.read_text() + source = entry_point.read_text(encoding = "utf-8") call_line = _ast_line_of_configure_call(source) compat_line = _ast_line_of_platform_compat_import(source) assert call_line < compat_line, ( diff --git a/studio/backend/tests/test_data_recipe_seed.py b/studio/backend/tests/test_data_recipe_seed.py index 58bbd24061..1b6fe27bfc 100644 --- a/studio/backend/tests/test_data_recipe_seed.py +++ b/studio/backend/tests/test_data_recipe_seed.py @@ -11,7 +11,7 @@ import pytest def _seed_route_source() -> str: return ( Path(__file__).resolve().parent.parent / "routes" / "data_recipe" / "seed.py" - ).read_text() + ).read_text(encoding = "utf-8") def test_seed_inspect_load_kwargs_disables_remote_code_execution(): diff --git a/studio/backend/tests/test_desktop_auth.py b/studio/backend/tests/test_desktop_auth.py index 591d44b736..b2180d4357 100644 --- a/studio/backend/tests/test_desktop_auth.py +++ b/studio/backend/tests/test_desktop_auth.py @@ -123,7 +123,7 @@ def test_ensure_default_admin_does_not_recreate_bootstrap_for_existing_admin(): def test_ensure_default_admin_loads_existing_bootstrap_after_restart(monkeypatch): created = storage.ensure_default_admin() - bootstrap_pw = storage._BOOTSTRAP_PW_PATH.read_text().strip() + bootstrap_pw = storage._BOOTSTRAP_PW_PATH.read_text(encoding = "utf-8").strip() monkeypatch.setattr(storage, "_bootstrap_password", None) created_again = storage.ensure_default_admin() @@ -136,12 +136,12 @@ def test_ensure_default_admin_loads_existing_bootstrap_after_restart(monkeypatch def test_ensure_default_admin_does_not_generate_for_empty_existing_bootstrap(): seed_user() - storage._BOOTSTRAP_PW_PATH.write_text(" \n") + storage._BOOTSTRAP_PW_PATH.write_text(" \n", encoding = "utf-8") created = storage.ensure_default_admin() assert created is False - assert storage._BOOTSTRAP_PW_PATH.read_text() == " \n" + assert storage._BOOTSTRAP_PW_PATH.read_text(encoding = "utf-8") == " \n" assert storage.get_bootstrap_password() is None @@ -649,7 +649,7 @@ def test_desktop_auth_provision_has_bounded_timeout(): rs_path = ( Path(__file__).resolve().parents[3] / "studio" / "src-tauri" / "src" / "desktop_auth.rs" ) - src = rs_path.read_text() + src = rs_path.read_text(encoding = "utf-8") start = src.index("async fn provision_desktop_auth(") depth = 0 body_start = src.index("{", start) diff --git a/studio/backend/tests/test_gguf_load_cache_reuse.py b/studio/backend/tests/test_gguf_load_cache_reuse.py index 0ab998af39..ccbe50bcb9 100644 --- a/studio/backend/tests/test_gguf_load_cache_reuse.py +++ b/studio/backend/tests/test_gguf_load_cache_reuse.py @@ -809,7 +809,9 @@ class TestLoadHubDownloadExclusion: asyncio.run(scenario()) def test_load_marker_precedes_hub_guard_and_unload(self): - source = (Path(__file__).resolve().parent.parent / "routes" / "inference.py").read_text() + source = (Path(__file__).resolve().parent.parent / "routes" / "inference.py").read_text( + encoding = "utf-8" + ) # _load_model_impl has more than one `if config.is_gguf:`, so anchor on # the branch that actually owns the load marker rather than the first # one in the file, which belongs to an earlier check. @@ -832,7 +834,7 @@ class TestLoadHubDownloadExclusion: ) llama_source = ( Path(__file__).resolve().parent.parent / "core" / "inference" / "llama_cpp.py" - ).read_text() + ).read_text(encoding = "utf-8") assert "@_with_gguf_load_marker\n def load_model(" in llama_source def _capture_hub_guard_require_mmproj( diff --git a/studio/backend/tests/test_host_defaults.py b/studio/backend/tests/test_host_defaults.py index 5c7129bc65..b5caba7573 100644 --- a/studio/backend/tests/test_host_defaults.py +++ b/studio/backend/tests/test_host_defaults.py @@ -64,7 +64,7 @@ def test_run_server_default_host_is_loopback(): 0.0.0.0 exposes the service on all interfaces; loopback is the least-permissive default. Users needing network access pass -H 0.0.0.0. """ - source = _RUN_PY.read_text() + source = _RUN_PY.read_text(encoding = "utf-8") defaults = _parse_function_param_defaults(source, "run_server") assert "host" in defaults, "run_server() must have a 'host' parameter with a default" host_default = defaults["host"] @@ -81,7 +81,7 @@ def test_argparse_default_host_is_loopback(): When run.py is invoked directly (python run.py), the argparse default must match the function default so direct execution is equally safe. """ - source = _RUN_PY.read_text() + source = _RUN_PY.read_text(encoding = "utf-8") host_default = _parse_argparse_add_argument_default(source, "--host") assert host_default is not None, "Could not find add_argument('--host', ...) in run.py" assert ( diff --git a/studio/backend/tests/test_mcp_servers.py b/studio/backend/tests/test_mcp_servers.py index c5c37f098f..731823c292 100644 --- a/studio/backend/tests/test_mcp_servers.py +++ b/studio/backend/tests/test_mcp_servers.py @@ -599,7 +599,9 @@ def test_tool_xml_strip_handles_hyphenated_function_names(): from core.inference.tool_call_parser import _DEEPSEEK_OPEN_RE_SRC as _DS_OPEN_SRC - src = (Path(__file__).resolve().parent.parent / "routes/inference.py").read_text() + src = (Path(__file__).resolve().parent.parent / "routes/inference.py").read_text( + encoding = "utf-8" + ) m = _re.search(r"_TOOL_XML_RE = _re\.compile\((.*?)\n\)", src, _re.DOTALL) assert m, "could not extract _TOOL_XML_RE" ns: dict = {"_re": _re, "_DS_OPEN_SRC": _DS_OPEN_SRC} diff --git a/studio/backend/tests/test_mlx_training_worker_config.py b/studio/backend/tests/test_mlx_training_worker_config.py index 14fc0933d0..5dde69648f 100644 --- a/studio/backend/tests/test_mlx_training_worker_config.py +++ b/studio/backend/tests/test_mlx_training_worker_config.py @@ -86,7 +86,9 @@ def test_mlx_studio_rejects_unknown_scheduler(): def test_mlx_studio_keeps_hf_style_tokenizer_dual_purpose(): - source = (Path(__file__).resolve().parents[1] / "core" / "training" / "worker.py").read_text() + source = (Path(__file__).resolve().parents[1] / "core" / "training" / "worker.py").read_text( + encoding = "utf-8" + ) assert "tokenizer = tokenizer" in source assert "processor = tokenizer if is_vlm else None" not in source @@ -96,7 +98,9 @@ def test_mlx_wandb_run_config_excludes_subject_and_secrets(): # The MLX W&B run config uploads the whole config minus a sensitive set. The owner's # subject (authenticated username / API-key id) must be filtered alongside the secrets, # otherwise it lands in W&B run config even though DB history already strips it. - source = (Path(__file__).resolve().parents[1] / "core" / "training" / "worker.py").read_text() + source = (Path(__file__).resolve().parents[1] / "core" / "training" / "worker.py").read_text( + encoding = "utf-8" + ) assert ( '_wandb_sensitive = {"hf_token", "wandb_token", "s3_config", "subject"}' in source diff --git a/studio/backend/tests/test_mtp_vram_budget.py b/studio/backend/tests/test_mtp_vram_budget.py index 694d60cfc6..6c8b74fc54 100644 --- a/studio/backend/tests/test_mtp_vram_budget.py +++ b/studio/backend/tests/test_mtp_vram_budget.py @@ -320,7 +320,7 @@ class TestFitContextWithMtp: def _fit_backend(self, kv_per_token = 325_000): b = _make_backend() b._can_estimate_kv = lambda: True - b._estimate_kv_cache_bytes = lambda n, _t = None, **_k: (0 if n <= 0 else n * kv_per_token) + b._estimate_kv_cache_bytes = lambda n, _t = None, **_k: 0 if n <= 0 else n * kv_per_token return b def test_overhead_fn_lowers_context(self): @@ -347,19 +347,23 @@ class TestFitContextWithMtp: 131072, avail_mib, model, - mtp_overhead_fn = lambda c: b._estimate_mtp_overhead_bytes( - c, draft_cache_type_k = "f16", draft_cache_type_v = "f16" - ) - or 0, + mtp_overhead_fn = lambda c: ( + b._estimate_mtp_overhead_bytes( + c, draft_cache_type_k = "f16", draft_cache_type_v = "f16" + ) + or 0 + ), ) q4 = b._fit_context_to_vram( 131072, avail_mib, model, - mtp_overhead_fn = lambda c: b._estimate_mtp_overhead_bytes( - c, draft_cache_type_k = "q4_0", draft_cache_type_v = "q4_0" - ) - or 0, + mtp_overhead_fn = lambda c: ( + b._estimate_mtp_overhead_bytes( + c, draft_cache_type_k = "q4_0", draft_cache_type_v = "q4_0" + ) + or 0 + ), ) assert 0 < q4 == f16 @@ -818,9 +822,9 @@ class TestExtraArgsMtpDetection: # helper, or an env-driven tensor server (or its layer downgrade) is # needlessly reloaded (#6312). Read from disk (importing routes.inference # drags in heavy deps). - routes_src = ( - Path(__file__).resolve().parent.parent / "routes" / "inference.py" - ).read_text() + routes_src = (Path(__file__).resolve().parent.parent / "routes" / "inference.py").read_text( + encoding = "utf-8" + ) start = routes_src.index("def _request_matches_loaded_settings") end = routes_src.index("\ndef ", start + 1) body = "".join(routes_src[start:end].split()) @@ -832,9 +836,9 @@ class TestExtraArgsMtpDetection: def test_route_matcher_retries_after_drafter_not_found(self): # drafter_not_found must not report "already loaded" or the reload never # retries the download (#6459). Read source: importing routes pulls deps. - routes_src = ( - Path(__file__).resolve().parent.parent / "routes" / "inference.py" - ).read_text() + routes_src = (Path(__file__).resolve().parent.parent / "routes" / "inference.py").read_text( + encoding = "utf-8" + ) start = routes_src.index("def _request_matches_loaded_settings") end = routes_src.index("\ndef ", start + 1) body = "".join(routes_src[start:end].split()) @@ -990,7 +994,7 @@ def test_qwen36_class_regression_picks_lower_ctx_with_mtp(): strictly lower one once the MTP draft reserve is accounted for.""" b = _make_backend() b._can_estimate_kv = lambda: True - b._estimate_kv_cache_bytes = lambda n, _t = None, **_k: (0 if n <= 0 else int(n * 66_000)) + b._estimate_kv_cache_bytes = lambda n, _t = None, **_k: 0 if n <= 0 else int(n * 66_000) avail_mib = 24_000 model = int(17.9 * GIB) # UD-Q4_K_XL weights no_mtp = b._fit_context_to_vram(262144, avail_mib, model) diff --git a/studio/backend/tests/test_native_context_length.py b/studio/backend/tests/test_native_context_length.py index de1ca0649e..9417b4c751 100644 --- a/studio/backend/tests/test_native_context_length.py +++ b/studio/backend/tests/test_native_context_length.py @@ -374,7 +374,7 @@ class TestRouteCompleteness: def _load_source(self): """Read routes/inference.py source once.""" routes_path = Path(__file__).resolve().parent.parent / "routes" / "inference.py" - self._source = routes_path.read_text() + self._source = routes_path.read_text(encoding = "utf-8") def _find_construction_blocks(self, class_name: str) -> list[str]: """Extract all code blocks that construct a given response class.""" diff --git a/studio/backend/tests/test_native_template_trust_remote_code.py b/studio/backend/tests/test_native_template_trust_remote_code.py index 60dc80f64c..b61a3eb111 100644 --- a/studio/backend/tests/test_native_template_trust_remote_code.py +++ b/studio/backend/tests/test_native_template_trust_remote_code.py @@ -170,7 +170,9 @@ def test_backend_model_info_persists_trust_remote_code(): """Both backends must store ``trust_remote_code`` on their per-model info dict so ``render_native_template`` can source the consent value. Guards against the read landing on a key ``load_model`` never sets (which would silently no-op the fix).""" - inf = (Path(_BACKEND_DIR) / "core" / "inference" / "inference.py").read_text() - mlx = (Path(_BACKEND_DIR) / "core" / "inference" / "mlx_inference.py").read_text() + inf = (Path(_BACKEND_DIR) / "core" / "inference" / "inference.py").read_text(encoding = "utf-8") + mlx = (Path(_BACKEND_DIR) / "core" / "inference" / "mlx_inference.py").read_text( + encoding = "utf-8" + ) assert '"trust_remote_code": trust_remote_code,' in inf assert '"trust_remote_code": trust_remote_code,' in mlx diff --git a/studio/backend/tests/test_offline_inference_parent.py b/studio/backend/tests/test_offline_inference_parent.py index bd0014ea64..3e9f09bb2f 100644 --- a/studio/backend/tests/test_offline_inference_parent.py +++ b/studio/backend/tests/test_offline_inference_parent.py @@ -205,7 +205,7 @@ class TestTrainingWorkerProbeNoGlobalTimeout: import re from pathlib import Path - src = Path(_BACKEND_DIR, "core", "training", "worker.py").read_text() + src = Path(_BACKEND_DIR, "core", "training", "worker.py").read_text(encoding = "utf-8") m = re.search( r'if\s+"HF_HUB_OFFLINE"\s+not\s+in\s+os\.environ\s*:.*?' r"print\([^)]*HF_HUB_OFFLINE=1[^)]*\)", diff --git a/studio/backend/tests/test_recommended_folders_has_model.py b/studio/backend/tests/test_recommended_folders_has_model.py index 647d5dd3db..c034824ba0 100644 --- a/studio/backend/tests/test_recommended_folders_has_model.py +++ b/studio/backend/tests/test_recommended_folders_has_model.py @@ -32,7 +32,7 @@ def _load_has_downloaded_model(): """Return the real ``_dir_has_downloaded_model`` (plus its ``_safe_is_dir`` and ``_is_weight_bin`` deps, and the ``_WEIGHT_BIN_PREFIXES`` constant the latter reads) without importing the heavy module.""" - tree = ast.parse(_models_src.read_text()) + tree = ast.parse(_models_src.read_text(encoding = "utf-8")) wanted = {"_safe_is_dir", "_dir_has_downloaded_model", "_is_weight_bin"} body = [] for node in tree.body: diff --git a/studio/backend/tests/test_recommended_folders_permission.py b/studio/backend/tests/test_recommended_folders_permission.py index b65695ad93..4f0becf08d 100644 --- a/studio/backend/tests/test_recommended_folders_permission.py +++ b/studio/backend/tests/test_recommended_folders_permission.py @@ -36,7 +36,7 @@ _models_src = _backend_root / "routes" / "models.py" def _load_safe_is_dir(): """Return the real ``_safe_is_dir`` from routes/models.py without importing the dependency-laden module.""" - tree = ast.parse(_models_src.read_text()) + tree = ast.parse(_models_src.read_text(encoding = "utf-8")) fn = next( node for node in tree.body diff --git a/studio/backend/tests/test_sandbox_tools.py b/studio/backend/tests/test_sandbox_tools.py index 853a5a84ab..1a55c6298d 100644 --- a/studio/backend/tests/test_sandbox_tools.py +++ b/studio/backend/tests/test_sandbox_tools.py @@ -558,24 +558,24 @@ class TestSandboxCpuRlimitDefault: """Pin the default so a regression below 600s without opt-in is caught.""" def test_default_cpu_s_is_600(self): - src = (_BACKEND_ROOT / "core" / "inference" / "tools.py").read_text() + src = (_BACKEND_ROOT / "core" / "inference" / "tools.py").read_text(encoding = "utf-8") assert 'UNSLOTH_STUDIO_SANDBOX_CPU_S", "600"' in src def test_clone_newnet_removed(self): - src = (_BACKEND_ROOT / "core" / "inference" / "tools.py").read_text() + src = (_BACKEND_ROOT / "core" / "inference" / "tools.py").read_text(encoding = "utf-8") assert "_libc.unshare(0x40000000)" not in src # Explanatory comment retained. assert "CLONE_NEWNET" in src def test_nofile_env_tunable(self): - src = (_BACKEND_ROOT / "core" / "inference" / "tools.py").read_text() + src = (_BACKEND_ROOT / "core" / "inference" / "tools.py").read_text(encoding = "utf-8") # Parity with the other rlimits: must come from the env, not be hardcoded. assert "UNSLOTH_STUDIO_SANDBOX_NOFILE" in src class TestMaxBodyDefault: def test_default_is_500_mb(self): - src = (_BACKEND_ROOT / "utils" / "upload_limits.py").read_text() + src = (_BACKEND_ROOT / "utils" / "upload_limits.py").read_text(encoding = "utf-8") assert "DEFAULT_UPLOAD_LIMIT_MB = 500" in src assert "UNSLOTH_STUDIO_MAX_BODY_MB" in src diff --git a/studio/backend/tests/test_security_gate_consistency.py b/studio/backend/tests/test_security_gate_consistency.py index b5f1069f12..0c0367e979 100644 --- a/studio/backend/tests/test_security_gate_consistency.py +++ b/studio/backend/tests/test_security_gate_consistency.py @@ -43,7 +43,7 @@ def test_capability_probes_thread_the_hf_token(): offenders = [] for path in _iter_caller_files(): try: - tree = ast.parse(path.read_text()) + tree = ast.parse(path.read_text(encoding = "utf-8")) except SyntaxError: continue for node in ast.walk(tree): @@ -60,7 +60,7 @@ def test_capability_probes_thread_the_hf_token(): def test_gguf_trust_remote_code_reported_inert_not_from_yaml(): """GGUF never executes auto_map, so requires_trust_remote_code is reported via the resolver or False, never the raw YAML bool() (the round-6 regression).""" - src = (_BACKEND / "routes" / "inference.py").read_text() + src = (_BACKEND / "routes" / "inference.py").read_text(encoding = "utf-8") assert "requires_trust_remote_code = bool(" not in src, ( "Report requires_trust_remote_code via _resolve_loaded_trust_remote_code " "(non-GGUF) or set it False (GGUF); never bool(inference_config.get(...))." @@ -70,7 +70,7 @@ def test_gguf_trust_remote_code_reported_inert_not_from_yaml(): def test_capability_detection_caches_are_token_aware(): """Every capability cache is keyed by (model, token_fingerprint) so an unauthenticated miss cannot poison a later authenticated lookup (the audio-cache regression).""" - src = (_BACKEND / "utils" / "models" / "model_config.py").read_text() + src = (_BACKEND / "utils" / "models" / "model_config.py").read_text(encoding = "utf-8") offenders = [] for line in src.splitlines(): stripped = line.strip() @@ -93,7 +93,7 @@ def test_malware_and_consent_gates_cover_the_lora_base(): ] offenders = [] for rel in gated_workers: - src = (_BACKEND / rel).read_text() + src = (_BACKEND / rel).read_text(encoding = "utf-8") runs_gate = "evaluate_file_security(" in src or "evaluate_remote_code_consent" in src resolves_base = "get_base_model_from_lora_identifier(" in src or "base_model" in src if runs_gate and not resolves_base: @@ -107,7 +107,7 @@ def test_rag_embedding_path_runs_the_malware_gate(): or a flagged repo loads unscanned (bypassing the normal model-load protections).""" offenders = [] for rel in ("routes/settings.py", "core/rag/embeddings.py"): - if "evaluate_file_security(" not in (_BACKEND / rel).read_text(): + if "evaluate_file_security(" not in (_BACKEND / rel).read_text(encoding = "utf-8"): offenders.append( f"{rel} loads/persists an embedding model without evaluate_file_security" ) diff --git a/studio/backend/tests/test_ssm_runtime.py b/studio/backend/tests/test_ssm_runtime.py index bb0caa2887..b95747e56c 100644 --- a/studio/backend/tests/test_ssm_runtime.py +++ b/studio/backend/tests/test_ssm_runtime.py @@ -401,13 +401,13 @@ def test_hip_uv_source_build_uses_no_cache(monkeypatch): def test_inference_worker_calls_ensure_ssm_runtime(): - src = (_BACKEND / "core" / "inference" / "worker.py").read_text() + src = (_BACKEND / "core" / "inference" / "worker.py").read_text(encoding = "utf-8") assert "from utils.ssm_runtime import ensure_ssm_runtime" in src assert "ensure_ssm_runtime(" in src def test_inference_worker_skips_ssm_on_mlx_and_checks_lora_base(): - src = (_BACKEND / "core" / "inference" / "worker.py").read_text() + src = (_BACKEND / "core" / "inference" / "worker.py").read_text(encoding = "utf-8") # MLX (Apple Silicon) must not try to build CUDA/ROCm SSM kernels. assert 'getattr(backend, "device", None) != "mlx"' in src # A LoRA load must also check its base model, not just the adapter id. @@ -417,12 +417,12 @@ def test_inference_worker_skips_ssm_on_mlx_and_checks_lora_base(): def test_inference_worker_resolves_remote_lora_base_pre_import(): # A remote LoRA's base (from the Hub adapter_config.json) must be resolved before the # transformers import so its SSM kernels are pre-installed, not too late in _handle_load. - src = (_BACKEND / "core" / "inference" / "worker.py").read_text() + src = (_BACKEND / "core" / "inference" / "worker.py").read_text(encoding = "utf-8") assert "_remote_lora_base" in src def test_inference_worker_tiers_on_base_and_gates_lora_base_only(): - src = (_BACKEND / "core" / "inference" / "worker.py").read_text() + src = (_BACKEND / "core" / "inference" / "worker.py").read_text(encoding = "utf-8") # Tier activation runs on the resolved base, not the raw adapter id (remote-LoRA fix). assert "_activate_transformers_version(_base" in src # The gate only adds a genuine LoRA base, never a full fine-tune's recorded (unloaded) base. @@ -432,7 +432,7 @@ def test_inference_worker_tiers_on_base_and_gates_lora_base_only(): def test_inference_worker_probes_base_for_ssm_kernels(): # Both the pre-import path and _handle_load must derive SSM targets from a real model id # via ssm_probe_identifier, not the raw adapter id / local checkpoint path. - src = (_BACKEND / "core" / "inference" / "worker.py").read_text() + src = (_BACKEND / "core" / "inference" / "worker.py").read_text(encoding = "utf-8") assert src.count("ssm_probe_identifier(") >= 2 @@ -484,7 +484,7 @@ def test_pre_import_gate_is_transformers_free(): def test_pre_import_gate_skips_subdir_computation(): # The worker's pre-import preflight must call the gate with compute_subdirs=False so it # never imports model_config/transformers before the SSM kernels are installed. - src = (_BACKEND / "core" / "inference" / "worker.py").read_text() + src = (_BACKEND / "core" / "inference" / "worker.py").read_text(encoding = "utf-8") assert "compute_subdirs = False" in src @@ -506,7 +506,7 @@ def test_security_gates_run_before_ssm_install(): # The SSM install is name-based and can source-build native packages, so a malware / # blocked-code model must be refused first -- in both the pre-import path and _handle_load. import ast - tree = ast.parse((_BACKEND / "core" / "inference" / "worker.py").read_text()) + tree = ast.parse((_BACKEND / "core" / "inference" / "worker.py").read_text(encoding = "utf-8")) for fn in ("run_inference_process", "_handle_load"): gates = _call_linenos(tree, fn, "_run_security_gates") ssm = _call_linenos(tree, fn, "_ensure_ssm_kernels") diff --git a/studio/backend/tests/test_studio_api.py b/studio/backend/tests/test_studio_api.py index 087c00b648..13dfccde20 100644 --- a/studio/backend/tests/test_studio_api.py +++ b/studio/backend/tests/test_studio_api.py @@ -403,9 +403,9 @@ def test_openai_tools_stream(base_url: str, api_key: str): ) assert status == 200, f"Expected 200, got {status}" assert len(chunks) > 0, "No SSE chunks received" - assert _final_finish_reason(chunks) == "tool_calls", ( - f"Expected final finish_reason='tool_calls', got " f"{_final_finish_reason(chunks)!r}" - ) + assert ( + _final_finish_reason(chunks) == "tool_calls" + ), f"Expected final finish_reason='tool_calls', got {_final_finish_reason(chunks)!r}" assembled = _collect_streamed_tool_calls(chunks) assert len(assembled) >= 1, "No tool_calls reassembled from stream" first = assembled[0] @@ -486,16 +486,16 @@ def test_openai_sdk_tool_calling(base_url: str, api_key: str): tool_choice = "required", stream = False, ) - assert resp.choices[0].finish_reason == "tool_calls", ( - f"Expected finish_reason='tool_calls', got " f"{resp.choices[0].finish_reason!r}" - ) + assert ( + resp.choices[0].finish_reason == "tool_calls" + ), f"Expected finish_reason='tool_calls', got {resp.choices[0].finish_reason!r}" tool_calls = resp.choices[0].message.tool_calls assert tool_calls and len(tool_calls) >= 1, "No tool_calls from SDK" tc = tool_calls[0] assert tc.function.name == "get_weather" parsed = json.loads(tc.function.arguments) assert "city" in parsed - print(f" PASS openai SDK tool calling: " f"tool={tc.function.name}, args={parsed}") + print(f" PASS openai SDK tool calling: tool={tc.function.name}, args={parsed}") def test_invalid_key_rejected(base_url: str): @@ -783,12 +783,17 @@ def _start_server(model: str, variant: str | None) -> tuple[subprocess.Popen, st cmd.extend(["--gguf-variant", variant]) LOG_FILE.parent.mkdir(parents = True, exist_ok = True) - log_fh = open(LOG_FILE, "w") + log_fh = open(LOG_FILE, "w", encoding = "utf-8") + # The child writes to this descriptor itself, so the parent's encoding does + # not transcode anything: tell the child to emit utf-8 or the reads below + # decode its locale bytes as utf-8 and raise on the first non-ASCII glyph. + child_env = {**os.environ, "PYTHONIOENCODING": "utf-8", "PYTHONUTF8": "1"} proc = subprocess.Popen( cmd, stdout = log_fh, stderr = subprocess.STDOUT, preexec_fn = os.setsid, + env = child_env, ) # Wait for the banner containing the API key @@ -798,16 +803,16 @@ def _start_server(model: str, variant: str | None) -> tuple[subprocess.Popen, st time.sleep(2) if proc.poll() is not None: log_fh.flush() - log_text = LOG_FILE.read_text() + log_text = LOG_FILE.read_text(encoding = "utf-8") raise RuntimeError(f"Server exited early (code {proc.returncode}):\n{log_text[-2000:]}") - log_text = LOG_FILE.read_text() + log_text = LOG_FILE.read_text(encoding = "utf-8") m = re.search(r"API Key:\s+(sk-unsloth-[a-f0-9]+)", log_text) if m: api_key = m.group(1) break if not api_key: - log_text = LOG_FILE.read_text() + log_text = LOG_FILE.read_text(encoding = "utf-8") _kill_server(proc) raise RuntimeError(f"Timed out waiting for API key in server output:\n{log_text[-2000:]}") diff --git a/studio/backend/tests/test_tool_call_parser_strict.py b/studio/backend/tests/test_tool_call_parser_strict.py index 02f63c41a2..0bf627e8aa 100644 --- a/studio/backend/tests/test_tool_call_parser_strict.py +++ b/studio/backend/tests/test_tool_call_parser_strict.py @@ -64,9 +64,7 @@ class TestFunctionStyleTrailingText: # The real closing is the last one; the literal inside # the code argument must survive (rfind, not the first match). text = ( - "" - 'print("")' - " all done" + 'print("") all done' ) call = _only(text) assert call == {"name": "python", "arguments": {"code": 'print("")'}} @@ -146,9 +144,7 @@ class TestParityWithJsonStyle: class TestGemmaNativeStyle: def test_closed_native_call_with_trailing_prose_is_accepted(self): - text = ( - '<|tool_call>call:terminal{command:"ls -la",workdir:"."}' " running it now" - ) + text = '<|tool_call>call:terminal{command:"ls -la",workdir:"."} running it now' calls = parse_tool_calls_from_text(text, allow_incomplete = False) assert len(calls) == 1 assert calls[0]["function"]["name"] == "terminal" @@ -792,7 +788,7 @@ def test_tool_call_parser_declares_future_annotations_for_py39_import(): from pathlib import Path src = ( Path(__file__).resolve().parent.parent / "core" / "inference" / "tool_call_parser.py" - ).read_text() + ).read_text(encoding = "utf-8") assert "from __future__ import annotations" in src @@ -1069,8 +1065,7 @@ class TestBareJsonOuterOverXmlLiteral: def test_bare_json_code_arg_quoting_function_xml(self): text = ( - '{"name": "python", "arguments": ' - '{"code": "run() # ls"}}' + '{"name": "python", "arguments": {"code": "run() # ls"}}' ) calls = parse_tool_calls_from_text(text, enabled_tool_names = {"python"}) assert [c["function"]["name"] for c in calls] == ["python"] @@ -1300,8 +1295,7 @@ class TestLeadingWrapperlessGemmaOverEmbeddedMarkers: def test_leading_gemma_wins_over_quoted_xml_literal(self): text = ( - 'call:web_search{query:"explain ' - '{"name":"evil","arguments":{}}"}' + 'call:web_search{query:"explain {"name":"evil","arguments":{}}"}' ) calls = parse_tool_calls_from_text(text, enabled_tool_names = {"web_search", "evil"}) assert [c["function"]["name"] for c in calls] == ["web_search"] diff --git a/studio/backend/tests/test_tool_xml_strip.py b/studio/backend/tests/test_tool_xml_strip.py index f7792a2a71..941d9d044a 100644 --- a/studio/backend/tests/test_tool_xml_strip.py +++ b/studio/backend/tests/test_tool_xml_strip.py @@ -21,7 +21,7 @@ if _BACKEND_DIR not in sys.path: # Extract the regex from source (routes module needs heavy stubbing to import). import re as _re -_src = (Path(_BACKEND_DIR) / "routes" / "inference.py").read_text() +_src = (Path(_BACKEND_DIR) / "routes" / "inference.py").read_text(encoding = "utf-8") _m = _re.search(r"_TOOL_XML_RE = _re\.compile\((.*?)\n\)", _src, _re.DOTALL) assert _m, "could not extract _TOOL_XML_RE source" # The lazy ``(.*?)\n\)`` could grab a shorter expression if an arm is ever wrapped; diff --git a/studio/backend/tests/test_tp_vision_regression.py b/studio/backend/tests/test_tp_vision_regression.py index d1372ca415..5dfc38f9af 100644 --- a/studio/backend/tests/test_tp_vision_regression.py +++ b/studio/backend/tests/test_tp_vision_regression.py @@ -450,7 +450,7 @@ def test_fallback_hint_uses_effective_tensor_request_not_just_toggle(): """Tensor intent keys off _effective_tensor_parallel (toggle + extras + env), not just the toggle, so extra/env-driven tensor users keep multi-GPU (#6659).""" route = Path(_BACKEND_DIR) / "routes" / "inference.py" - src = route.read_text() + src = route.read_text(encoding = "utf-8") idx = src.find("_tensor_intent_overall = _effective_tensor_parallel(") assert idx != -1, "the GGUF load closure must compute tensor intent" block = src[idx : idx + 300] @@ -482,7 +482,7 @@ def test_preserved_fallback_carried_across_non_drop_reload(): gated on the same model loaded, so a ctx-only reload keeps multi-GPU but a model switch / explicit drop doesn't inherit it (#6659).""" route = Path(_BACKEND_DIR) / "routes" / "inference.py" - src = route.read_text() + src = route.read_text(encoding = "utf-8") idx = src.find("_tensor_intent_overall = _effective_tensor_parallel(") assert idx != -1 block = src[idx : idx + 400] @@ -499,7 +499,7 @@ def test_same_model_guard_checks_path_and_variant(): repo), so a reload keeps the carry-forward and a different variant doesn't inherit the prior one's preserved tensor intent (#6659).""" route = Path(_BACKEND_DIR) / "routes" / "inference.py" - src = route.read_text() + src = route.read_text(encoding = "utf-8") idx = src.find("_same_model_loaded = (") assert idx != -1 block = src[idx : idx + 1300] @@ -748,7 +748,7 @@ def test_explicit_tensor_drop_uses_shared_helper_in_both_readers(): _is_explicit_tensor_drop, so they agree on what counts as a drop -- a reload for an unrelated extra still carries the preserved intent rather than collapsing to one GPU (Codex #6659).""" - src = (Path(_BACKEND_DIR) / "routes" / "inference.py").read_text() + src = (Path(_BACKEND_DIR) / "routes" / "inference.py").read_text(encoding = "utf-8") # Dedup reader (the preserved-fallback reload guard). assert "layer_preserves_tensor_intent and _is_explicit_tensor_drop(request)" in src # Load carry-forward reader feeds the same decision into the carry-forward. diff --git a/studio/backend/tests/test_training_raw_support.py b/studio/backend/tests/test_training_raw_support.py index fb3cffc91e..49281605e6 100644 --- a/studio/backend/tests/test_training_raw_support.py +++ b/studio/backend/tests/test_training_raw_support.py @@ -163,13 +163,13 @@ class TestTrainingRawSupport(unittest.TestCase): def test_route_forwards_all_grad_clipping_fields(self): # The HTTP route builds the config dict by hand; a schema field that # is not forwarded here is silently dropped for REST callers. - source = (_BACKEND_ROOT / "routes" / "training.py").read_text() + source = (_BACKEND_ROOT / "routes" / "training.py").read_text(encoding = "utf-8") self.assertIn('"max_grad_norm": request.max_grad_norm', source) self.assertIn('"max_grad_value": request.max_grad_value', source) self.assertIn('"max_grad_leaf_norm": request.max_grad_leaf_norm', source) def test_mlx_worker_falls_back_init_seeds_to_random_seed(self): - source = (_BACKEND_ROOT / "core" / "training" / "worker.py").read_text() + source = (_BACKEND_ROOT / "core" / "training" / "worker.py").read_text(encoding = "utf-8") # random_seed itself is normalized first so explicit None coming # from a raw / backend caller does not propagate through the chain. @@ -198,7 +198,7 @@ class TestTrainingRawSupport(unittest.TestCase): self.assertIn("seed = random_seed,", source) def test_mlx_worker_preserves_null_max_grad_value_for_trainer_default(self): - source = (_BACKEND_ROOT / "core" / "training" / "worker.py").read_text() + source = (_BACKEND_ROOT / "core" / "training" / "worker.py").read_text(encoding = "utf-8") # None must survive to the MLX trainer so it picks its own runtime # default, and any other value must coerce to float without @@ -251,7 +251,7 @@ class TestTrainingRawSupport(unittest.TestCase): # unsloth-zoo update. Until that floor is in place, the # worker must gate them so releases that predate those fields can # still construct MLXTrainingConfig without TypeError. - source = (_BACKEND_ROOT / "core" / "training" / "worker.py").read_text() + source = (_BACKEND_ROOT / "core" / "training" / "worker.py").read_text(encoding = "utf-8") self.assertIn( 'getattr(MLXTrainingConfig, "__dataclass_fields__", {})', diff --git a/studio/backend/tests/test_transformers_version.py b/studio/backend/tests/test_transformers_version.py index acb2ec449b..7926ace1d3 100644 --- a/studio/backend/tests/test_transformers_version.py +++ b/studio/backend/tests/test_transformers_version.py @@ -2672,7 +2672,7 @@ class TestLatestTierForces16Bit: def _read(self, rel): backend_dir = Path(__file__).resolve().parent.parent - return (backend_dir / rel).read_text() + return (backend_dir / rel).read_text(encoding = "utf-8") def test_worker_guard_present(self): src = self._read("core/inference/worker.py") diff --git a/studio/backend/tests/test_yaml_trust_remote_code_removed.py b/studio/backend/tests/test_yaml_trust_remote_code_removed.py index 9578f08420..fa313cf0fa 100644 --- a/studio/backend/tests/test_yaml_trust_remote_code_removed.py +++ b/studio/backend/tests/test_yaml_trust_remote_code_removed.py @@ -19,7 +19,7 @@ _MODEL_DEFAULTS = _CONFIGS / "model_defaults" def test_no_model_default_yaml_sets_trust_remote_code(): offenders = [] for f in _MODEL_DEFAULTS.rglob("*.yaml"): - doc = yaml.safe_load(f.read_text()) or {} + doc = yaml.safe_load(f.read_text(encoding = "utf-8")) or {} if not isinstance(doc, dict): continue for section, body in doc.items(): @@ -37,7 +37,7 @@ def test_no_model_default_yaml_has_empty_or_none_section(): # A bare `inference:` header (no keys) parses to None and crashes the .get() loaders. offenders = [] for f in _MODEL_DEFAULTS.rglob("*.yaml"): - doc = yaml.safe_load(f.read_text()) + doc = yaml.safe_load(f.read_text(encoding = "utf-8")) if not isinstance(doc, dict): offenders.append(f"{f.relative_to(_CONFIGS)} (not a mapping)") continue @@ -96,7 +96,7 @@ def test_all_model_yamls_load_for_training_and_inference(): def test_base_templates_have_no_trust_remote_code(): for name in ("full_finetune.yaml", "lora_text.yaml", "vision_lora.yaml"): - doc = yaml.safe_load((_CONFIGS / name).read_text()) or {} + doc = yaml.safe_load((_CONFIGS / name).read_text(encoding = "utf-8")) or {} flat = yaml.safe_dump(doc) assert "trust_remote_code" not in flat, f"{name} should not set trust_remote_code" diff --git a/tests/python/test_cpo_processor_text_tokenizer.py b/tests/python/test_cpo_processor_text_tokenizer.py index 69316e042d..9440cba28a 100644 --- a/tests/python/test_cpo_processor_text_tokenizer.py +++ b/tests/python/test_cpo_processor_text_tokenizer.py @@ -40,7 +40,7 @@ def _registrations(source): def test_cpo_registration_matches_orpo(): - regs = _registrations(open(RL_PATH).read()) + regs = _registrations(open(RL_PATH, encoding = "utf-8").read()) shared = {"orpo_trainer_text_tokenizer", "orpo_trainer_processor_pad_token"} assert shared <= set(regs.get("orpo_trainer", [])) assert shared <= set(regs.get("cpo_trainer", [])) @@ -48,7 +48,7 @@ def test_cpo_registration_matches_orpo(): def _load_pad_rewriter(): """Exec orpo_trainer_processor_pad_token (+ _PAD_FALLBACK) without importing unsloth.""" - tree = ast.parse(open(RL_PATH).read()) + tree = ast.parse(open(RL_PATH, encoding = "utf-8").read()) nodes = [] for n in tree.body: if isinstance(n, ast.Assign) and any( diff --git a/tests/python/test_dpo_vision_processor_passthrough.py b/tests/python/test_dpo_vision_processor_passthrough.py index a320cab935..f9f8cee24f 100644 --- a/tests/python/test_dpo_vision_processor_passthrough.py +++ b/tests/python/test_dpo_vision_processor_passthrough.py @@ -11,7 +11,7 @@ RL_PATH = os.path.join(REPO_ROOT, "unsloth", "models", "rl_replacements.py") def _load_helpers(): - src = open(RL_PATH).read() + src = open(RL_PATH, encoding = "utf-8").read() tree = ast.parse(src) import torch as _torch diff --git a/tests/python/test_e2e_no_torch_sandbox.py b/tests/python/test_e2e_no_torch_sandbox.py index bb61af462d..3e46f4145e 100644 --- a/tests/python/test_e2e_no_torch_sandbox.py +++ b/tests/python/test_e2e_no_torch_sandbox.py @@ -193,7 +193,7 @@ class TestBeforeAfterImportChain: mm = types.ModuleType('model_mappings') mm.MODEL_TO_TEMPLATE_MAPPER = {{}} sys.modules['model_mappings'] = mm - source = open({str(before_file)!r}).read() + source = open({str(before_file)!r}, encoding = "utf-8").read() source = source.replace('from .format_detection import', 'from format_detection import') source = source.replace('from .model_mappings import', 'from model_mappings import') exec(source) @@ -215,7 +215,7 @@ class TestBeforeAfterImportChain: loggers = types.ModuleType('loggers') loggers.get_logger = lambda n: None sys.modules['loggers'] = loggers - exec(open({str(before_file)!r}).read()) + exec(open({str(before_file)!r}, encoding = "utf-8").read()) """) result = _run_in_sandbox(no_torch_venv, code) assert result.returncode != 0, "BEFORE data_collators.py should crash without torch" @@ -284,7 +284,7 @@ class TestBeforeAfterImportChain: it = types.ModuleType('iterable') it.is_streaming_dataset = lambda *a, **k: False sys.modules['iterable'] = it - source = open({str(CHAT_TEMPLATES)!r}).read() + source = open({str(CHAT_TEMPLATES)!r}, encoding = "utf-8").read() source = source.replace('from .format_detection import', 'from format_detection import') source = source.replace('from .model_mappings import', 'from model_mappings import') source = source.replace('from .iterable import', 'from iterable import') @@ -304,7 +304,7 @@ class TestBeforeAfterImportChain: loggers = types.ModuleType('loggers') loggers.get_logger = lambda n: None sys.modules['loggers'] = loggers - exec(open({str(DATA_COLLATORS)!r}).read()) + exec(open({str(DATA_COLLATORS)!r}, encoding = "utf-8").read()) print("OK") """) result = _run_in_sandbox(no_torch_venv, code) @@ -382,7 +382,7 @@ class TestDataclassInstantiation: loggers = types.ModuleType('loggers') loggers.get_logger = lambda n: None sys.modules['loggers'] = loggers - exec(open({str(DATA_COLLATORS)!r}).read()) + exec(open({str(DATA_COLLATORS)!r}, encoding = "utf-8").read()) obj = DataCollatorSpeechSeq2SeqWithPadding(processor=None) assert obj.processor is None print("OK") @@ -397,7 +397,7 @@ class TestDataclassInstantiation: loggers = types.ModuleType('loggers') loggers.get_logger = lambda n: None sys.modules['loggers'] = loggers - exec(open({str(DATA_COLLATORS)!r}).read()) + exec(open({str(DATA_COLLATORS)!r}, encoding = "utf-8").read()) obj = DeepSeekOCRDataCollator(processor=None) assert obj.processor is None assert obj.max_length == 2048 @@ -414,7 +414,7 @@ class TestDataclassInstantiation: loggers = types.ModuleType('loggers') loggers.get_logger = lambda n: None sys.modules['loggers'] = loggers - exec(open({str(DATA_COLLATORS)!r}).read()) + exec(open({str(DATA_COLLATORS)!r}, encoding = "utf-8").read()) obj = VLMDataCollator(processor=None) assert obj.processor is None assert obj.max_length == 2048 @@ -441,7 +441,7 @@ class TestDataclassInstantiation: it.is_streaming_dataset = lambda *a, **k: False sys.modules['iterable'] = it ns = {{}} - source = open({str(CHAT_TEMPLATES)!r}).read() + source = open({str(CHAT_TEMPLATES)!r}, encoding = "utf-8").read() source = source.replace('from .format_detection import', 'from format_detection import') source = source.replace('from .model_mappings import', 'from model_mappings import') source = source.replace('from .iterable import', 'from iterable import') @@ -473,7 +473,7 @@ class TestEdgeCasesBrokenTorch: code = textwrap.dedent(f"""\ import sys sys.path.insert(0, {str(sandbox_dir)!r}) - exec(open({str(sandbox_dir / 'data_collators.py')!r}).read()) + exec(open({str(sandbox_dir / 'data_collators.py')!r}, encoding = "utf-8").read()) obj = DataCollatorSpeechSeq2SeqWithPadding(processor=None) print("OK: data_collators works despite broken torch on sys.path") """) @@ -495,7 +495,7 @@ class TestEdgeCasesBrokenTorch: code = textwrap.dedent(f"""\ import sys sys.path.insert(0, {str(sandbox_dir)!r}) - source = open({str(HARDWARE_PY)!r}).read() + source = open({str(HARDWARE_PY)!r}, encoding = "utf-8").read() ns = {{'__name__': '__test__'}} exec(source, ns) result = ns['detect_hardware']() @@ -530,7 +530,7 @@ class TestEdgeCasesBrokenTorch: code = textwrap.dedent(f"""\ import sys sys.path.insert(0, {str(sandbox_dir)!r}) - source = open({str(HARDWARE_PY)!r}).read() + source = open({str(HARDWARE_PY)!r}, encoding = "utf-8").read() ns = {{'__name__': '__test__'}} exec(source, ns) result = ns['detect_hardware']() @@ -559,7 +559,7 @@ class TestEdgeCasesBrokenTorch: sys.modules['iterable'] = it ns = {{}} - source = open({str(CHAT_TEMPLATES)!r}).read() + source = open({str(CHAT_TEMPLATES)!r}, encoding = "utf-8").read() source = source.replace('from .format_detection import', 'from format_detection import') source = source.replace('from .model_mappings import', 'from model_mappings import') source = source.replace('from .iterable import', 'from iterable import') @@ -604,7 +604,7 @@ class TestHardwareDetectionNoTorch: code = textwrap.dedent(f"""\ import sys sys.path.insert(0, {str(sandbox_dir)!r}) - source = open({str(HARDWARE_PY)!r}).read() + source = open({str(HARDWARE_PY)!r}, encoding = "utf-8").read() ns = {{'__name__': '__test__'}} exec(source, ns) device = ns['detect_hardware']() @@ -624,7 +624,7 @@ class TestHardwareDetectionNoTorch: code = textwrap.dedent(f"""\ import sys sys.path.insert(0, {str(sandbox_dir)!r}) - source = open({str(HARDWARE_PY)!r}).read() + source = open({str(HARDWARE_PY)!r}, encoding = "utf-8").read() ns = {{'__name__': '__test__'}} exec(source, ns) versions = ns['get_package_versions']() @@ -651,7 +651,7 @@ class TestHardwareDetectionNoTorch: code = textwrap.dedent(f"""\ import sys sys.path.insert(0, {str(sandbox_dir)!r}) - source = open({str(hw_sandbox / 'hardware.py')!r}).read() + source = open({str(hw_sandbox / 'hardware.py')!r}, encoding = "utf-8").read() ns = {{'__name__': '__test__'}} exec(source, ns) assert callable(ns['detect_hardware']) diff --git a/tests/python/test_fast_language_model_text_only.py b/tests/python/test_fast_language_model_text_only.py index fcdeb49bc3..08e5cdf0dc 100644 --- a/tests/python/test_fast_language_model_text_only.py +++ b/tests/python/test_fast_language_model_text_only.py @@ -14,7 +14,7 @@ UTILS_PATH = REPO_ROOT / "unsloth" / "models" / "_utils.py" def _source(path): - return path.read_text() + return path.read_text(encoding = "utf-8") def _class_method(tree, class_name, method_name): diff --git a/tests/python/test_fast_model_config_passthrough.py b/tests/python/test_fast_model_config_passthrough.py index b2ba3d2eef..6ab941478e 100644 --- a/tests/python/test_fast_model_config_passthrough.py +++ b/tests/python/test_fast_model_config_passthrough.py @@ -12,7 +12,7 @@ LLAMA_PATH = REPO_ROOT / "unsloth" / "models" / "llama.py" def _source(path): - return path.read_text() + return path.read_text(encoding = "utf-8") def _class_method(tree, class_name, method_name): diff --git a/tests/python/test_gpu_init_ldconfig_guard.py b/tests/python/test_gpu_init_ldconfig_guard.py index 248bb84faa..986dfcec8b 100644 --- a/tests/python/test_gpu_init_ldconfig_guard.py +++ b/tests/python/test_gpu_init_ldconfig_guard.py @@ -17,13 +17,13 @@ def _find_geteuid_guard(tree: ast.AST): def test_gpu_init_has_geteuid_guard(): - tree = ast.parse(GPU_INIT.read_text()) + tree = ast.parse(GPU_INIT.read_text(encoding = "utf-8")) guard = _find_geteuid_guard(tree) assert guard is not None, "_gpu_init.py must guard ldconfig recovery on os.geteuid()" def test_ldconfig_calls_only_inside_geteuid_guard(): - src = GPU_INIT.read_text() + src = GPU_INIT.read_text(encoding = "utf-8") tree = ast.parse(src) guard = _find_geteuid_guard(tree) assert guard is not None @@ -39,6 +39,6 @@ def test_ldconfig_calls_only_inside_geteuid_guard(): def test_non_root_branch_warns_when_bnb_present(): - src = GPU_INIT.read_text() + src = GPU_INIT.read_text(encoding = "utf-8") assert "elif bnb is not None" in src assert "sudo ldconfig" in src diff --git a/tests/python/test_grpo_ddp_model_config.py b/tests/python/test_grpo_ddp_model_config.py index 5af31f65b8..23614d3add 100644 --- a/tests/python/test_grpo_ddp_model_config.py +++ b/tests/python/test_grpo_ddp_model_config.py @@ -9,7 +9,7 @@ SOURCE_PATH = os.path.join(REPO_ROOT, "unsloth", "models", "rl_replacements.py") def _read_source() -> str: - with open(SOURCE_PATH, "r") as fh: + with open(SOURCE_PATH, "r", encoding = "utf-8") as fh: return fh.read() diff --git a/tests/python/test_orpo_processor_text_tokenizer.py b/tests/python/test_orpo_processor_text_tokenizer.py index b507a9e808..84bfe60bb0 100644 --- a/tests/python/test_orpo_processor_text_tokenizer.py +++ b/tests/python/test_orpo_processor_text_tokenizer.py @@ -10,7 +10,7 @@ RL_PATH = os.path.join(REPO_ROOT, "unsloth", "models", "rl_replacements.py") def _load_orpo_rewriter(name = "orpo_trainer_text_tokenizer"): - src = open(RL_PATH).read() + src = open(RL_PATH, encoding = "utf-8").read() tree = ast.parse(src) ns = {"re": re} # Materialise sibling module-level _-prefixed assignments the rewriter may reference. diff --git a/tests/python/test_pad_token_fix.py b/tests/python/test_pad_token_fix.py index 5f2a29a323..c19c6969ce 100644 --- a/tests/python/test_pad_token_fix.py +++ b/tests/python/test_pad_token_fix.py @@ -21,7 +21,7 @@ WANTED = { def _load_pad_helpers(): """Exec only the pad-token helpers with a stub logger (no heavy imports).""" - tree = ast.parse(open(TOK_PATH).read()) + tree = ast.parse(open(TOK_PATH, encoding = "utf-8").read()) nodes = [] for node in tree.body: if isinstance(node, ast.Assign): diff --git a/tests/python/test_studio_import_no_torch.py b/tests/python/test_studio_import_no_torch.py index f551519de9..48b62fd99d 100644 --- a/tests/python/test_studio_import_no_torch.py +++ b/tests/python/test_studio_import_no_torch.py @@ -148,7 +148,7 @@ class TestDataCollatorsNoTorchVenv: loggers = types.ModuleType('loggers') loggers.get_logger = lambda n: None sys.modules['loggers'] = loggers - exec(open({str(DATA_COLLATORS)!r}).read()) + exec(open({str(DATA_COLLATORS)!r}, encoding = "utf-8").read()) print("OK: exec succeeded") """) result = subprocess.run( @@ -168,7 +168,7 @@ class TestDataCollatorsNoTorchVenv: loggers = types.ModuleType('loggers') loggers.get_logger = lambda n: None sys.modules['loggers'] = loggers - exec(open({str(DATA_COLLATORS)!r}).read()) + exec(open({str(DATA_COLLATORS)!r}, encoding = "utf-8").read()) obj = DataCollatorSpeechSeq2SeqWithPadding(processor=None) assert obj.processor is None, "processor should be None" print("OK: DataCollatorSpeechSeq2SeqWithPadding instantiated") @@ -190,7 +190,7 @@ class TestDataCollatorsNoTorchVenv: loggers = types.ModuleType('loggers') loggers.get_logger = lambda n: None sys.modules['loggers'] = loggers - exec(open({str(DATA_COLLATORS)!r}).read()) + exec(open({str(DATA_COLLATORS)!r}, encoding = "utf-8").read()) obj = DeepSeekOCRDataCollator(processor=None) assert obj.processor is None, "processor should be None" assert obj.max_length == 2048, "default max_length should be 2048" @@ -212,7 +212,7 @@ class TestDataCollatorsNoTorchVenv: loggers = types.ModuleType('loggers') loggers.get_logger = lambda n: None sys.modules['loggers'] = loggers - exec(open({str(DATA_COLLATORS)!r}).read()) + exec(open({str(DATA_COLLATORS)!r}, encoding = "utf-8").read()) obj = VLMDataCollator(processor=None) assert obj.processor is None assert obj.mask_input_tokens is True, "default mask_input_tokens should be True" @@ -259,7 +259,7 @@ class TestChatTemplatesNoTorchVenv: sys.modules['iterable'] = iterable # Read and transform the source: replace relative imports with absolute - source = open({str(CHAT_TEMPLATES)!r}).read() + source = open({str(CHAT_TEMPLATES)!r}, encoding = "utf-8").read() source = source.replace('from .format_detection import', 'from format_detection import') source = source.replace('from .model_mappings import', 'from model_mappings import') source = source.replace('from .iterable import', 'from iterable import') @@ -305,7 +305,7 @@ class TestChatTemplatesNoTorchVenv: sys.modules['iterable'] = iterable ns = {{}} - source = open({str(CHAT_TEMPLATES)!r}).read() + source = open({str(CHAT_TEMPLATES)!r}, encoding = "utf-8").read() source = source.replace('from .format_detection import', 'from format_detection import') source = source.replace('from .model_mappings import', 'from model_mappings import') source = source.replace('from .iterable import', 'from iterable import') @@ -402,7 +402,7 @@ class TestFormatConversionNoTorchVenv: sys.modules['utils.hardware'] = hardware_mod # Read and exec format_conversion.py - source = open({str(FORMAT_CONVERSION)!r}).read() + source = open({str(FORMAT_CONVERSION)!r}, encoding = "utf-8").read() source = source.replace('from .format_detection import', 'from format_detection import') source = source.replace('from .iterable import', 'from iterable import') ns = {{'__name__': '__test__'}} @@ -463,7 +463,7 @@ class TestFormatConversionNoTorchVenv: sys.modules['utils'] = utils_mod sys.modules['utils.hardware'] = hardware_mod - source = open({str(FORMAT_CONVERSION)!r}).read() + source = open({str(FORMAT_CONVERSION)!r}, encoding = "utf-8").read() source = source.replace('from .format_detection import', 'from format_detection import') source = source.replace('from .iterable import', 'from iterable import') ns = {{'__name__': '__test__'}} @@ -517,7 +517,7 @@ class TestNegativeControls: loggers = types.ModuleType('loggers') loggers.get_logger = lambda n: None sys.modules['loggers'] = loggers - exec(open({temp_file!r}).read()) + exec(open({temp_file!r}, encoding = "utf-8").read()) """) result = subprocess.run( [no_torch_venv, "-c", code], diff --git a/tests/python/test_v100_fullft_precision.py b/tests/python/test_v100_fullft_precision.py index c8ca769d45..49abcb4ad6 100644 --- a/tests/python/test_v100_fullft_precision.py +++ b/tests/python/test_v100_fullft_precision.py @@ -29,7 +29,7 @@ RL_PY = Path(__file__).resolve().parents[2] / "unsloth" / "models" / "rl.py" def _extract_mixed_precision_code() -> str: - lines = RL_PY.read_text().split("\n") + lines = RL_PY.read_text(encoding = "utf-8").split("\n") try: start = next(i for i, l in enumerate(lines) if "mixed_precision = (" in l) except StopIteration: diff --git a/tests/python/test_vision_lora_targeting.py b/tests/python/test_vision_lora_targeting.py index 0a27569efd..bed26aa297 100644 --- a/tests/python/test_vision_lora_targeting.py +++ b/tests/python/test_vision_lora_targeting.py @@ -37,7 +37,7 @@ def test_vlm_lora_regex_respects_language_only_with_explicit_targets(): def test_fast_vision_model_wraps_explicit_targets_when_layer_filters_are_used(): - source = Path("unsloth/models/vision.py").read_text() + source = Path("unsloth/models/vision.py").read_text(encoding = "utf-8") assert "target_modules = get_peft_regex(" in source assert "target_modules = list(target_modules)" in source diff --git a/tests/saving/test_fix_sentencepiece_gguf_robustness.py b/tests/saving/test_fix_sentencepiece_gguf_robustness.py index 9c61ca4067..2b9cc87a60 100644 --- a/tests/saving/test_fix_sentencepiece_gguf_robustness.py +++ b/tests/saving/test_fix_sentencepiece_gguf_robustness.py @@ -82,7 +82,7 @@ def test_entry_with_non_int_id_is_skipped(tmp_path): def test_save_py_except_clause_is_broad_exception(): - with open(_SAVE_PY) as f: + with open(_SAVE_PY, encoding = "utf-8") as f: tree = ast.parse(f.read()) for node in ast.walk(tree): if isinstance(node, ast.FunctionDef) and node.name == "unsloth_save_pretrained_gguf": @@ -102,7 +102,7 @@ def test_save_py_except_clause_is_broad_exception(): def test_tokenizer_utils_uses_import_protobuf_fallback_pattern(): - with open(_TOK_PY) as f: + with open(_TOK_PY, encoding = "utf-8") as f: src = f.read() tree = ast.parse(src) for node in ast.walk(tree): diff --git a/tests/security/test_scan_packages.py b/tests/security/test_scan_packages.py index 48e6da5f66..2608494b42 100644 --- a/tests/security/test_scan_packages.py +++ b/tests/security/test_scan_packages.py @@ -35,9 +35,11 @@ def test_fixture_bytes_are_deterministic(tmp_path): rebuild_dir = tmp_path / "rebuild" rebuild_dir.mkdir() # The build helper writes to its own dir; copy + patch HERE. - builder_src = (FIXTURES / "_build.py").read_text() + builder_src = (FIXTURES / "_build.py").read_text(encoding = "utf-8") rebuilt_helper = rebuild_dir / "_build.py" - rebuilt_helper.write_text(builder_src) + # builder_src came out of a checked-in file, so it carries whatever + # non-ASCII that file holds and cp1252 cannot encode it back out. + rebuilt_helper.write_text(builder_src, encoding = "utf-8") # Run with SOURCE_DATE_EPOCH=0 and HERE override via a shim. shim = rebuild_dir / "run.py" shim.write_text( @@ -1260,7 +1262,7 @@ def test_committed_baseline_suppresses_known_but_not_a_new_payload(): import json baseline_path = REPO_ROOT / "scripts" / "scan_packages_baseline.json" - entries = json.loads(baseline_path.read_text())["entries"] + entries = json.loads(baseline_path.read_text(encoding = "utf-8"))["entries"] target = next( e for e in entries @@ -1296,7 +1298,7 @@ def test_committed_baseline_entries_all_carry_evidence_hash(): import json baseline_path = REPO_ROOT / "scripts" / "scan_packages_baseline.json" - entries = json.loads(baseline_path.read_text())["entries"] + entries = json.loads(baseline_path.read_text(encoding = "utf-8"))["entries"] assert entries, "committed baseline should not be empty" missing = [ f"{e['package']}:{e['file']}:{e['check']}" for e in entries if not e.get("evidence_hash") diff --git a/tests/studio/install/test_llama_pr_force_and_source.py b/tests/studio/install/test_llama_pr_force_and_source.py index 4ff8c349c3..8d89660924 100644 --- a/tests/studio/install/test_llama_pr_force_and_source.py +++ b/tests/studio/install/test_llama_pr_force_and_source.py @@ -346,7 +346,7 @@ class TestSourcePatternsSh: @pytest.fixture(autouse = True) def _load_source(self): - self.content = SETUP_SH.read_text() + self.content = SETUP_SH.read_text(encoding = "utf-8") def test_has_default_pr_force(self): assert '_DEFAULT_LLAMA_PR_FORCE=""' in self.content @@ -412,7 +412,7 @@ class TestSourcePatternsPs1: @pytest.fixture(autouse = True) def _load_source(self): - self.content = SETUP_PS1.read_text() + self.content = SETUP_PS1.read_text(encoding = "utf-8") def test_has_default_pr_force(self): assert '$DefaultLlamaPrForce = ""' in self.content diff --git a/tests/studio/install/test_managed_node_runtime.py b/tests/studio/install/test_managed_node_runtime.py index 17c7e3e60f..251cb0ffcd 100644 --- a/tests/studio/install/test_managed_node_runtime.py +++ b/tests/studio/install/test_managed_node_runtime.py @@ -125,7 +125,7 @@ def test_resolve_falls_back_to_managed_when_no_system(monkeypatch, tmp_path): monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path)) managed = nr.managed_node_binary() managed.parent.mkdir(parents = True, exist_ok = True) - managed.write_text("#!/bin/sh\necho v24.17.0\n") + managed.write_text("#!/bin/sh\necho v24.17.0\n", encoding = "utf-8") monkeypatch.setattr(nr.shutil, "which", lambda name: None) monkeypatch.setattr(nr, "_node_version_ok", lambda exe: str(exe) == str(managed)) assert nr.resolve_node_executable() == str(managed) @@ -136,7 +136,7 @@ def test_resolve_prefers_managed_over_unsuitable_system(monkeypatch, tmp_path): monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path)) managed = nr.managed_node_binary() managed.parent.mkdir(parents = True, exist_ok = True) - managed.write_text("fake") + managed.write_text("fake", encoding = "utf-8") monkeypatch.setattr(nr.shutil, "which", lambda name: "/old/node") monkeypatch.setattr(nr, "_node_version_ok", lambda exe: str(exe) == str(managed)) assert nr.resolve_node_executable() == str(managed) @@ -167,7 +167,7 @@ def test_negative_result_is_not_cached(monkeypatch, tmp_path): managed = nr.managed_node_binary() managed.parent.mkdir(parents = True, exist_ok = True) - managed.write_text("now-installed") + managed.write_text("now-installed", encoding = "utf-8") monkeypatch.setattr(nr, "_node_version_ok", lambda exe: str(exe) == str(managed)) assert nr.resolve_node_executable() == str(managed) diff --git a/tests/studio/install/test_pr4562_bugfixes.py b/tests/studio/install/test_pr4562_bugfixes.py index 0d2b092924..4e555d76c6 100644 --- a/tests/studio/install/test_pr4562_bugfixes.py +++ b/tests/studio/install/test_pr4562_bugfixes.py @@ -617,7 +617,7 @@ class TestSourceCodePatterns: def test_setup_sh_no_rm_before_prereq_check(self): """rm -rf must appear AFTER cmake/git checks, not before.""" - content = SETUP_SH.read_text() + content = SETUP_SH.read_text(encoding = "utf-8") # Anchor on the source-build cmake check block. idx_block = content.find("command -v cmake") assert idx_block != -1 @@ -630,7 +630,7 @@ class TestSourceCodePatterns: def test_setup_sh_clone_uses_branch_tag(self): """git clone in source-build should use --branch via the clone args array.""" - content = SETUP_SH.read_text() + content = SETUP_SH.read_text(encoding = "utf-8") assert "_CLONE_ARGS=(git clone --depth 1)" in content assert ( '_CLONE_ARGS+=(--branch "$_RESOLVED_SOURCE_REF")' in content @@ -642,7 +642,7 @@ class TestSourceCodePatterns: def test_setup_sh_source_build_uses_helper_latest_tag_only(self): """Shell source fallback should only use helper latest-tag resolution.""" - content = SETUP_SH.read_text() + content = SETUP_SH.read_text(encoding = "utf-8") assert "--resolve-source-build" not in content assert "--resolve-install-tag" not in content assert '--resolve-llama-tag latest --published-repo "ggml-org/llama.cpp"' in content @@ -653,7 +653,7 @@ class TestSourceCodePatterns: def test_setup_sh_prebuilt_install_entrypoint(self): """Shell prebuilt path uses the helper install entrypoint, not the old releases-latest flow.""" - content = SETUP_SH.read_text() + content = SETUP_SH.read_text(encoding = "utf-8") assert "--resolve-install-tag" not in content assert "_HELPER_RELEASE_REPO}/releases/latest" not in content assert "ggml-org/llama.cpp/releases/latest" not in content @@ -663,7 +663,7 @@ class TestSourceCodePatterns: fork like every other host, so the release-repo decision is unconditional. Guards against a silent reintroduction of a ggml-org CPU routing branch. GPU usability detection (used for PyTorch / source decisions) must stay.""" - content = SETUP_SH.read_text() + content = SETUP_SH.read_text(encoding = "utf-8") assert '_HELPER_RELEASE_REPO="unslothai/llama.cpp"' in content assert '_HELPER_RELEASE_REPO="ggml-org/llama.cpp"' not in content # Usability gating (not routing) still distinguishes a hidden GPU. @@ -676,14 +676,14 @@ class TestSourceCodePatterns: def test_setup_sh_reports_installed_prebuilt_release(self): """Shell wrapper should report the installed prebuilt release from metadata.""" - content = SETUP_SH.read_text() + content = SETUP_SH.read_text(encoding = "utf-8") assert "UNSLOTH_PREBUILT_INFO.json" in content assert "installed release:" in content assert 'print_installed_llama_prebuilt_release "$LLAMA_CPP_DIR"' in content def test_setup_sh_macos_arm64_uses_metal_flags(self): """Apple Silicon source builds should explicitly enable Metal like upstream.""" - content = SETUP_SH.read_text() + content = SETUP_SH.read_text(encoding = "utf-8") assert "_IS_MACOS_ARM64=true" in content assert 'if [ "$_IS_MACOS_ARM64" = true ]; then' in content assert "-DGGML_METAL=ON" in content @@ -695,7 +695,7 @@ class TestSourceCodePatterns: def test_setup_sh_macos_metal_configure_has_cpu_fallback(self): """GPU configure/build failure retries a CPU build. Stays label-agnostic (PR #5826 generalised the Metal-only wording via $_FB_LABEL).""" - content = SETUP_SH.read_text() + content = SETUP_SH.read_text(encoding = "utf-8") assert "_TRY_METAL_CPU_FALLBACK=true" in content assert 'configure failed; retrying CPU build..." "$C_WARN"' in content assert 'build failed; retrying CPU build..." "$C_WARN"' in content @@ -714,7 +714,7 @@ class TestSourceCodePatterns: """PR #5826: a fresh CUDA toolkit's host-compiler whitelist lags distro gcc/clang (nvcc "#error -- unsupported GNU version"). setup.sh exports NVCC_PREPEND_FLAGS=-allow-unsupported-compiler via env, not CMAKE_ARGS (word-splitting safety).""" - content = SETUP_SH.read_text() + content = SETUP_SH.read_text(encoding = "utf-8") assert "-allow-unsupported-compiler" in content # Via NVCC_PREPEND_FLAGS (covers the configure-time probe too), not CMAKE_ARGS. assert "export NVCC_PREPEND_FLAGS=" in content @@ -726,7 +726,7 @@ class TestSourceCodePatterns: def test_setup_ps1_exports_allow_unsupported_compiler(self): """Windows parity for PR #5826: CUDA toolkit whitelist lags MSVC. setup.ps1 sets NVCC_PREPEND_FLAGS=-allow-unsupported-compiler in the CUDA branch via env, out of $CmakeArgs.""" - content = SETUP_PS1.read_text() + content = SETUP_PS1.read_text(encoding = "utf-8") assert "-allow-unsupported-compiler" in content # Via process env, not $CmakeArgs, so it reaches both the configure probe and `cmake --build`. assert "$env:NVCC_PREPEND_FLAGS" in content @@ -763,7 +763,7 @@ class TestSourceCodePatterns: def test_setup_sh_does_not_enable_metal_for_intel_macos(self): """Intel macOS should stay on the existing non-Metal path in this patch.""" - content = SETUP_SH.read_text() + content = SETUP_SH.read_text(encoding = "utf-8") assert 'if [ "$_IS_MACOS_ARM64" = true ]; then' in content assert ( 'Darwin" ] && { [ "$_HOST_MACHINE" = "arm64" ] || [ "$_HOST_MACHINE" = "aarch64" ]; }' @@ -778,20 +778,20 @@ class TestSourceCodePatterns: def test_setup_ps1_uses_checkout_b(self): """PS1 should use checkout -B, not checkout --force FETCH_HEAD.""" - content = SETUP_PS1.read_text() + content = SETUP_PS1.read_text(encoding = "utf-8") assert "checkout -B unsloth-llama-build" in content assert "checkout --force FETCH_HEAD" not in content def test_setup_ps1_clone_uses_branch_tag(self): """PS1 clone should use --branch with the resolved tag.""" - content = SETUP_PS1.read_text() + content = SETUP_PS1.read_text(encoding = "utf-8") assert "--branch" in content and "$ResolvedSourceRef" in content # The old commented-out clone line should be gone. assert "# git clone --depth 1 --branch" not in content def test_setup_ps1_no_git_pull(self): """PS1 should use fetch, not pull (which fails in detached HEAD).""" - content = SETUP_PS1.read_text() + content = SETUP_PS1.read_text(encoding = "utf-8") # No "git pull" in the source-build section (only valid on a branch). lines = content.splitlines() for i, line in enumerate(lines): @@ -800,18 +800,18 @@ class TestSourceCodePatterns: # Allowed elsewhere; fail only in the llama.cpp build section. context = "\n".join(lines[max(0, i - 5) : i + 5]) if "LlamaCppDir" in context: - pytest.fail(f"Found 'git pull' in llama.cpp build section at line {i+1}") + pytest.fail(f"Found 'git pull' in llama.cpp build section at line {i + 1}") def test_setup_ps1_prebuilt_install_entrypoint(self): """PS1 prebuilt path uses the helper install entrypoint, not the old releases-latest flow.""" - content = SETUP_PS1.read_text() + content = SETUP_PS1.read_text(encoding = "utf-8") assert "--resolve-install-tag" not in content assert "$HelperReleaseRepo/releases/latest" not in content assert "ggml-org/llama.cpp/releases/latest" not in content def test_setup_ps1_reports_installed_prebuilt_release(self): """PS1 wrapper should report the installed prebuilt release from metadata.""" - content = SETUP_PS1.read_text() + content = SETUP_PS1.read_text(encoding = "utf-8") assert "Get-InstalledLlamaPrebuiltRelease" in content assert "UNSLOTH_PREBUILT_INFO.json" in content assert "installed release:" in content @@ -822,7 +822,7 @@ class TestSourceCodePatterns: def test_setup_ps1_source_build_uses_helper_latest_tag_only(self): """PS1 source fallback should only use helper latest-tag resolution.""" - content = SETUP_PS1.read_text() + content = SETUP_PS1.read_text(encoding = "utf-8") assert "--resolve-source-build" not in content assert "--resolve-install-tag" not in content assert ( @@ -835,7 +835,7 @@ class TestSourceCodePatterns: def test_setup_ps1_prebuilt_install_disables_native_error_abort(self): """PS1 prebuilt install should not abort setup on helper stderr.""" - content = SETUP_PS1.read_text() + content = SETUP_PS1.read_text(encoding = "utf-8") install_idx = content.index("& python @prebuiltArgs 2>&1") block = content[max(0, install_idx - 800) : install_idx + 800] assert "$PSNativeCommandUseErrorActionPreference = $false" in block @@ -844,7 +844,7 @@ class TestSourceCodePatterns: def test_setup_ps1_helper_disables_error_action_abort(self): """Helper resolution should suppress terminating NativeCommandError on PS 5.1.""" - content = SETUP_PS1.read_text() + content = SETUP_PS1.read_text(encoding = "utf-8") helper_idx = content.index("function Invoke-LlamaHelper") block = content[helper_idx : helper_idx + 2200] assert "$previousErrorActionPreference = $ErrorActionPreference" in block @@ -853,19 +853,19 @@ class TestSourceCodePatterns: def test_setup_ps1_uses_local_tempfile_helper(self): """PS1 should not depend on New-TemporaryFile being available anywhere.""" - content = SETUP_PS1.read_text() + content = SETUP_PS1.read_text(encoding = "utf-8") assert "function New-UnslothTemporaryFile" in content assert "$resolveErrorLog = New-TemporaryFile" not in content def test_setup_ps1_find_nvcc_uses_version_sort_for_latest_toolkit(self): """The unconstrained nvcc fallback should not sort toolkit dirs lexicographically.""" - content = SETUP_PS1.read_text() + content = SETUP_PS1.read_text(encoding = "utf-8") assert "Sort-Object Name | Select-Object -Last 1" not in content assert "Sort-Object { [version]($_.Name -replace '^v','') } -Descending" in content def test_binary_env_linux_has_binary_parent(self): """The Linux branch of binary_env should include binary_path.parent.""" - content = MODULE_PATH.read_text() + content = MODULE_PATH.read_text(encoding = "utf-8") in_func = False in_linux = False found = False diff --git a/tests/studio/load_freeze/test_load_orchestrator.py b/tests/studio/load_freeze/test_load_orchestrator.py index 64503d0e05..a1f4caa309 100644 --- a/tests/studio/load_freeze/test_load_orchestrator.py +++ b/tests/studio/load_freeze/test_load_orchestrator.py @@ -441,7 +441,7 @@ def test_load_model_caches_audio_type_inside_serial_load_lock(): """Audio-type detection must run inside load_model under _serial_load_lock, else a concurrent /load can replace the backend mid-probe (review on #5669).""" f = _REPO_ROOT / "studio" / "backend" / "core" / "inference" / "llama_cpp.py" - text = f.read_text() + text = f.read_text(encoding = "utf-8") assert ( "with self._serial_load_lock" in text ), "LlamaCppBackend.load_model must hold self._serial_load_lock" @@ -462,7 +462,7 @@ def test_routes_inference_reads_cached_audio_type_not_calls_detect(): """routes/inference.py must read cached _audio_type/_is_audio, not call detect_audio_type / init_audio_codec directly (both moved into load_model).""" f = _REPO_ROOT / "studio" / "backend" / "routes" / "inference.py" - text = f.read_text() + text = f.read_text(encoding = "utf-8") assert "llama_backend.detect_audio_type(" not in text, ( "routes/inference.py should not call detect_audio_type directly; " "load_model already cached it under the lock." @@ -485,7 +485,7 @@ def test_no_other_async_route_calls_detect_audio_type_unwrapped(): # function helper is excluded below. pattern = re.compile(r"\b\w+\.detect_audio_type\s*\(") for path in routes_dir.rglob("*.py"): - for i, line in enumerate(path.read_text().splitlines(), start = 1): + for i, line in enumerate(path.read_text(encoding = "utf-8").splitlines(), start = 1): m = pattern.search(line) if not m: continue diff --git a/tests/studio/playwright_chat_ime_i18n.py b/tests/studio/playwright_chat_ime_i18n.py index 5bebf0a9e8..7883df16be 100644 --- a/tests/studio/playwright_chat_ime_i18n.py +++ b/tests/studio/playwright_chat_ime_i18n.py @@ -241,10 +241,12 @@ with sync_playwright() as p: # Source-level guard: grep the unmounted edit/compare composers' JSX for dir="auto". _repo_root = Path(__file__).resolve().parents[2] - _thread_src = ( - _repo_root / "studio/frontend/src/components/assistant-ui/thread.tsx" - ).read_text() - _shared_src = (_repo_root / "studio/frontend/src/features/chat/shared-composer.tsx").read_text() + _thread_src = (_repo_root / "studio/frontend/src/components/assistant-ui/thread.tsx").read_text( + encoding = "utf-8" + ) + _shared_src = (_repo_root / "studio/frontend/src/features/chat/shared-composer.tsx").read_text( + encoding = "utf-8" + ) _edit_idx = _thread_src.find("aui-edit-composer-input") if _edit_idx == -1 or 'dir="auto"' not in _thread_src[_edit_idx : _edit_idx + 600]: soft_fail('edit composer source is missing dir="auto"') diff --git a/tests/studio/playwright_chat_ui.py b/tests/studio/playwright_chat_ui.py index a06e559100..b182e66f01 100644 --- a/tests/studio/playwright_chat_ui.py +++ b/tests/studio/playwright_chat_ui.py @@ -98,7 +98,7 @@ def expected_default_model(): / "defaults.py" ) try: - tree = ast.parse(defaults_path.read_text()) + tree = ast.parse(defaults_path.read_text(encoding = "utf-8")) except Exception as exc: fail(f"could not read {defaults_path}: {exc}") models = None diff --git a/tests/studio/studio_api_smoke.py b/tests/studio/studio_api_smoke.py index d30bd11dca..ce55c06223 100644 --- a/tests/studio/studio_api_smoke.py +++ b/tests/studio/studio_api_smoke.py @@ -142,7 +142,7 @@ except Exception as exc: # GET / cross-origin must NOT leak the bootstrap password in the served HTML. boot_path = AUTH_DIR / ".bootstrap_password" if boot_path.exists(): - bootstrap_pw = boot_path.read_text().strip() + bootstrap_pw = boot_path.read_text(encoding = "utf-8").strip() if bootstrap_pw: req = urllib.request.Request( f"{BASE}/", diff --git a/tests/studio/test_auth_form_input_count.py b/tests/studio/test_auth_form_input_count.py index 75e6cfd1fb..aa7975cf10 100644 --- a/tests/studio/test_auth_form_input_count.py +++ b/tests/studio/test_auth_form_input_count.py @@ -51,7 +51,7 @@ def _conditional_extent(src: str) -> tuple[int, int]: def test_hasbootstrappassword_constant_is_derived_from_bootstrap_window_value(): """The guard must read from window.__UNSLOTH_BOOTSTRAP__, matching the backend's bootstrap-injection contract in studio/backend/main.py::_inject_bootstrap.""" - src = AUTH_FORM.read_text() + src = AUTH_FORM.read_text(encoding = "utf-8") assert "const hasBootstrapPassword = Boolean(window.__UNSLOTH_BOOTSTRAP__?.password);" in src, ( "hasBootstrapPassword constant missing or its derivation drifted; " "this is the gate that hides the Current password input on first boot" @@ -61,7 +61,7 @@ def test_hasbootstrappassword_constant_is_derived_from_bootstrap_window_value(): def test_exactly_one_hasBootstrapPassword_conditional_exists(): """Only one `!hasBootstrapPassword` JSX check is allowed; a second would split rendering into branches and likely hide or duplicate the New / Confirm inputs.""" - src = AUTH_FORM.read_text() + src = AUTH_FORM.read_text(encoding = "utf-8") count = src.count("!hasBootstrapPassword") assert count == 1, ( f"expected exactly one !hasBootstrapPassword usage, found {count}; " @@ -72,7 +72,7 @@ def test_exactly_one_hasBootstrapPassword_conditional_exists(): def test_current_password_input_is_inside_the_hasBootstrapPassword_conditional(): """`id="current-password"` must sit inside `{!hasBootstrapPassword && (...)}`, else it renders on first boot too, regressing the pre-#5490 UX that PR #5545 restores.""" - src = AUTH_FORM.read_text() + src = AUTH_FORM.read_text(encoding = "utf-8") s, e = _conditional_extent(src) idx = src.find('id="current-password"') assert idx != -1, "the Current password input was removed entirely" @@ -86,7 +86,7 @@ def test_current_password_input_is_inside_the_hasBootstrapPassword_conditional() def test_new_password_input_is_outside_the_hasBootstrapPassword_conditional(): """`id="new-password"` must sit outside `{!hasBootstrapPassword && (...)}`, else it disappears on admin-forced resets, regressing PR #5490.""" - src = AUTH_FORM.read_text() + src = AUTH_FORM.read_text(encoding = "utf-8") s, e = _conditional_extent(src) idx = src.find('id="new-password"') assert idx != -1, "the New password input was removed entirely" @@ -99,7 +99,7 @@ def test_new_password_input_is_outside_the_hasBootstrapPassword_conditional(): def test_confirm_password_input_is_outside_the_hasBootstrapPassword_conditional(): """Same as New password, for `id="confirm-password"`.""" - src = AUTH_FORM.read_text() + src = AUTH_FORM.read_text(encoding = "utf-8") s, e = _conditional_extent(src) idx = src.find('id="confirm-password"') assert idx != -1, "the Confirm password input was removed entirely" @@ -114,7 +114,7 @@ def test_change_password_jsx_declares_exactly_three_password_inputs(): """The change-password JSX block (`{!isLoginMode && (...)}`) must declare exactly current/new/confirm; a fourth would break the 2-input first-boot contract (the conditional only hides Current).""" - src = AUTH_FORM.read_text() + src = AUTH_FORM.read_text(encoding = "utf-8") start = src.find("{!isLoginMode && (") assert start != -1, ( "the change-password JSX subtree marker {!isLoginMode && (...)} " @@ -147,7 +147,7 @@ def test_change_password_jsx_declares_exactly_three_password_inputs(): def test_login_jsx_declares_exactly_one_password_input(): """The login JSX block (`isLoginMode && (...)`) must declare exactly one password input (the bootstrap password pasted from the CLI); a second breaks the per-mode matrix.""" - src = AUTH_FORM.read_text() + src = AUTH_FORM.read_text(encoding = "utf-8") start = src.find("{isLoginMode && (") assert start != -1, "the login JSX subtree marker is missing" depth = 1 @@ -163,18 +163,20 @@ def test_login_jsx_declares_exactly_one_password_input(): ids = re.findall(r'id="([a-z-]+)"', subtree) # Lock the count, not the spelling, so a rename does not falsely fail. pw_ids = [x for x in ids if "password" in x] - assert len(pw_ids) == 1, ( - f"login JSX must declare exactly one password-typed input; " f"found {pw_ids!r}" - ) + assert ( + len(pw_ids) == 1 + ), f"login JSX must declare exactly one password-typed input; found {pw_ids!r}" def test_auth_flow_routes_do_not_mount_global_settings(): - root = (FRONTEND / "app/routes/__root.tsx").read_text() + root = (FRONTEND / "app/routes/__root.tsx").read_text(encoding = "utf-8") assert "{!isAuthFlowRoute && }" in root assert "useSettingsDialogStore.getState().closeDialog();" in root assert "if (isAuthFlowRoute) return;" in root for route in ("login", "change-password", "onboarding"): - assert "isAuthFlow: true" in (FRONTEND / f"app/routes/{route}.tsx").read_text() + assert "isAuthFlow: true" in (FRONTEND / f"app/routes/{route}.tsx").read_text( + encoding = "utf-8" + ) def test_auth_redirect_targets_are_idempotent_and_concurrent(tmp_path: Path): @@ -190,7 +192,7 @@ def test_auth_redirect_targets_are_idempotent_and_concurrent(tmp_path: Path): pytest.skip("node --experimental-strip-types not available") source = ( - AUTH_API.read_text() + AUTH_API.read_text(encoding = "utf-8") .replace('from "@/lib/api-base"', 'from "./stubs.mjs"') .replace('from "./session"', 'from "./stubs.mjs"') ) diff --git a/tests/studio/test_cancel_atomicity.py b/tests/studio/test_cancel_atomicity.py index 142cc2247a..391ef043d7 100644 --- a/tests/studio/test_cancel_atomicity.py +++ b/tests/studio/test_cancel_atomicity.py @@ -9,7 +9,7 @@ from pathlib import Path SOURCE_PATH = Path(__file__).resolve().parents[2] / "studio" / "backend" / "routes" / "inference.py" -_SRC = SOURCE_PATH.read_text() +_SRC = SOURCE_PATH.read_text(encoding = "utf-8") _TREE = ast.parse(_SRC) diff --git a/tests/studio/test_cancel_id_wiring.py b/tests/studio/test_cancel_id_wiring.py index 651dfbf3de..fba0e814f6 100644 --- a/tests/studio/test_cancel_id_wiring.py +++ b/tests/studio/test_cancel_id_wiring.py @@ -13,10 +13,14 @@ from pathlib import Path WORKSPACE = Path(__file__).resolve().parents[2] -MODELS_SRC = (WORKSPACE / "studio/backend/models/inference.py").read_text() -ROUTES_SRC = (WORKSPACE / "studio/backend/routes/inference.py").read_text() -ADAPTER_SRC = (WORKSPACE / "studio/frontend/src/features/chat/api/chat-adapter.ts").read_text() -API_TYPES_SRC = (WORKSPACE / "studio/frontend/src/features/chat/types/api.ts").read_text() +MODELS_SRC = (WORKSPACE / "studio/backend/models/inference.py").read_text(encoding = "utf-8") +ROUTES_SRC = (WORKSPACE / "studio/backend/routes/inference.py").read_text(encoding = "utf-8") +ADAPTER_SRC = (WORKSPACE / "studio/frontend/src/features/chat/api/chat-adapter.ts").read_text( + encoding = "utf-8" +) +API_TYPES_SRC = (WORKSPACE / "studio/frontend/src/features/chat/types/api.ts").read_text( + encoding = "utf-8" +) def _find_class(tree: ast.AST, name: str) -> ast.ClassDef | None: diff --git a/tests/studio/test_chat_preset_builtin_invariants.py b/tests/studio/test_chat_preset_builtin_invariants.py index 3ca09dadda..31b2a2b733 100644 --- a/tests/studio/test_chat_preset_builtin_invariants.py +++ b/tests/studio/test_chat_preset_builtin_invariants.py @@ -40,13 +40,15 @@ def _require_node(): def _ensure_harness(): TEMP.mkdir(parents = True, exist_ok = True) (TEMP / "register.mjs").write_text( - "import { register } from 'node:module';\nregister('./loader.mjs', import.meta.url);\n" + "import { register } from 'node:module';\nregister('./loader.mjs', import.meta.url);\n", + encoding = "utf-8", ) (TEMP / "loader.mjs").write_text( "export function resolve(specifier, context, next) {\n" " if (specifier.endsWith('/types/runtime')) return next(specifier + '.ts', context);\n" " return next(specifier, context);\n" - "}\n" + "}\n", + encoding = "utf-8", ) @@ -54,7 +56,7 @@ def _run(script: str): _require_node() _ensure_harness() script_path = TEMP / "run.mts" - script_path.write_text(script) + script_path.write_text(script, encoding = "utf-8") env = dict(os.environ, NODE_NO_WARNINGS = "1") result = subprocess.run( [ diff --git a/tests/studio/test_chat_prompt_variables.py b/tests/studio/test_chat_prompt_variables.py index dcf318b5fa..6ef3b79ae9 100644 --- a/tests/studio/test_chat_prompt_variables.py +++ b/tests/studio/test_chat_prompt_variables.py @@ -6,7 +6,9 @@ from pathlib import Path WORKSPACE = Path(__file__).resolve().parents[2] -ADAPTER_SRC = (WORKSPACE / "studio/frontend/src/features/chat/api/chat-adapter.ts").read_text() +ADAPTER_SRC = (WORKSPACE / "studio/frontend/src/features/chat/api/chat-adapter.ts").read_text( + encoding = "utf-8" +) def _function_source(name: str) -> str: diff --git a/tests/studio/test_chat_response_details_ui_contract.py b/tests/studio/test_chat_response_details_ui_contract.py index 1183151b54..aa1a5cd965 100644 --- a/tests/studio/test_chat_response_details_ui_contract.py +++ b/tests/studio/test_chat_response_details_ui_contract.py @@ -20,14 +20,14 @@ CHAT_TAB_TSX = REPO / "studio/frontend/src/features/settings/tabs/chat-tab.tsx" def test_assistant_more_menu_exposes_response_details_action(): - src = THREAD_TSX.read_text() + src = THREAD_TSX.read_text(encoding = "utf-8") assert "MessageResponseDetailsSheet" in src assert "See response details" in src assert "setDetailsOpen(true)" in src def test_response_details_sheet_uses_unsloth_sheet_and_key_sections(): - src = DETAILS_TSX.read_text() + src = DETAILS_TSX.read_text(encoding = "utf-8") assert "SheetContent" in src assert "Response details" in src assert "MessageResponseModelBadge" in src @@ -45,17 +45,17 @@ def test_response_details_sheet_uses_unsloth_sheet_and_key_sections(): def test_response_model_badge_is_user_configurable_and_rendered_once_per_message(): - prefs_src = CHAT_PREFS_TS.read_text() - chat_tab_src = CHAT_TAB_TSX.read_text() - thread_src = THREAD_TSX.read_text() - reasoning_src = REASONING_TSX.read_text() + prefs_src = CHAT_PREFS_TS.read_text(encoding = "utf-8") + chat_tab_src = CHAT_TAB_TSX.read_text(encoding = "utf-8") + thread_src = THREAD_TSX.read_text(encoding = "utf-8") + reasoning_src = REASONING_TSX.read_text(encoding = "utf-8") assert "showResponseModel: boolean" in prefs_src assert "showResponseModel: false" in prefs_src assert "showResponseModel: saved?.showResponseModel ?? false" in prefs_src assert "Show response model" in chat_tab_src assert "setShowResponseModel" in chat_tab_src - details_src = DETAILS_TSX.read_text() + details_src = DETAILS_TSX.read_text(encoding = "utf-8") assert ( "aui-response-model-badge pointer-events-none relative inline-flex min-h-5" in details_src ) @@ -76,7 +76,7 @@ def test_response_model_badge_is_user_configurable_and_rendered_once_per_message def test_reasoning_keeps_streaming_height_cap_through_automatic_collapse(): - src = REASONING_TSX.read_text() + src = REASONING_TSX.read_text(encoding = "utf-8") assert "const [retainStreamingHeight, setRetainStreamingHeight]" in src assert "setRetainStreamingHeight(false)" in src @@ -91,7 +91,7 @@ def test_reasoning_clears_manual_open_on_a_new_stream(): isOpen is `(streaming && !dismissed) || manualOpen` and manualOpen is only settable while idle, so the new-stream reset has to clear it too. """ - src = REASONING_TSX.read_text() + src = REASONING_TSX.read_text(encoding = "utf-8") marker = "setDismissedWhileStreaming(false)" start = src.find(marker) @@ -101,7 +101,7 @@ def test_reasoning_clears_manual_open_on_a_new_stream(): def test_response_details_metadata_is_persisted_without_backend_schema_change(): - src = ADAPTER_TS.read_text() + src = ADAPTER_TS.read_text(encoding = "utf-8") assert "interface ResponseDetailsMetadata" in src assert "buildResponseDetails" in src assert "responseDetails: buildResponseDetails(finishedAt)" in src diff --git a/tests/studio/test_chat_title_generation.py b/tests/studio/test_chat_title_generation.py index 6a47cfbce4..3a8cbb95f9 100644 --- a/tests/studio/test_chat_title_generation.py +++ b/tests/studio/test_chat_title_generation.py @@ -41,7 +41,7 @@ def _balanced_block(src: str, anchor: str) -> str: def test_title_model_prompt_targets_conversation_topic(): block = _source_until( - RUNTIME_TSX.read_text(), + RUNTIME_TSX.read_text(encoding = "utf-8"), "async function generateTitleWithModel", "\nconst inflightTitleByKey", ) @@ -54,7 +54,7 @@ def test_title_model_prompt_targets_conversation_topic(): def test_title_model_payload_includes_optional_assistant_reply(): block = _source_until( - RUNTIME_TSX.read_text(), + RUNTIME_TSX.read_text(encoding = "utf-8"), "async function generateTitleWithModel", "\nconst inflightTitleByKey", ) @@ -71,7 +71,7 @@ def test_title_model_payload_includes_optional_assistant_reply(): def test_generate_title_passes_first_assistant_reply_after_first_user(): block = _balanced_block( - RUNTIME_TSX.read_text(), + RUNTIME_TSX.read_text(encoding = "utf-8"), "async generateTitle(remoteId", ) @@ -84,7 +84,7 @@ def test_generate_title_passes_first_assistant_reply_after_first_user(): def test_tool_call_only_first_assistant_still_uses_first_user_message(): - source = RUNTIME_TSX.read_text() + source = RUNTIME_TSX.read_text(encoding = "utf-8") extract_block = " ".join(_balanced_block(source, "function extractTextParts").split()) generate_block = " ".join(_balanced_block(source, "async generateTitle(remoteId").split()) @@ -104,7 +104,7 @@ def test_tool_call_only_first_assistant_still_uses_first_user_message(): def test_auto_title_disabled_uses_deterministic_user_text_fallback(): block = _balanced_block( - RUNTIME_TSX.read_text(), + RUNTIME_TSX.read_text(encoding = "utf-8"), "async generateTitle(remoteId", ) auto_title_off = _balanced_block(block, "if (!autoTitle)") @@ -114,7 +114,7 @@ def test_auto_title_disabled_uses_deterministic_user_text_fallback(): def test_model_failure_still_falls_back_to_user_text(): - source = RUNTIME_TSX.read_text() + source = RUNTIME_TSX.read_text(encoding = "utf-8") model_block = _source_until( source, "async function generateTitleWithModel", @@ -130,7 +130,7 @@ def test_model_failure_still_falls_back_to_user_text(): def test_title_normalizer_still_enforces_output_constraints(): block = _source_until( - RUNTIME_TSX.read_text(), + RUNTIME_TSX.read_text(encoding = "utf-8"), "async function generateTitleWithModel", "\nconst inflightTitleByKey", ) diff --git a/tests/studio/test_cli_run_alias.py b/tests/studio/test_cli_run_alias.py index 498ebbdf4d..98b5d4f76c 100644 --- a/tests/studio/test_cli_run_alias.py +++ b/tests/studio/test_cli_run_alias.py @@ -17,7 +17,7 @@ def _module_calls(source: str): def test_top_level_run_alias_registered(): """`app.command("run", ...)` must be invoked with studio_run as its target.""" - source = _CLI_INIT.read_text() + source = _CLI_INIT.read_text(encoding = "utf-8") # Find ``app.command("run", ...)`` call -- the decorator-call form. found_decorator_call = False @@ -46,7 +46,7 @@ def test_top_level_run_alias_registered(): def test_studio_run_imported_for_alias(): """The alias must wire up to the studio.run function, not redefine it.""" - source = _CLI_INIT.read_text() + source = _CLI_INIT.read_text(encoding = "utf-8") tree = ast.parse(source) has_import = False for node in ast.walk(tree): diff --git a/tests/studio/test_cli_studio_defaults.py b/tests/studio/test_cli_studio_defaults.py index a39956fab0..17ff23e66c 100644 --- a/tests/studio/test_cli_studio_defaults.py +++ b/tests/studio/test_cli_studio_defaults.py @@ -48,20 +48,19 @@ def _find_typer_option_default(source: str, func_name: str, long_option: str): def test_studio_default_host_is_loopback(): """`unsloth studio` (studio_default) --host default must be 127.0.0.1.""" - source = _STUDIO_CMD_PY.read_text() + source = _STUDIO_CMD_PY.read_text(encoding = "utf-8") host_default = _find_typer_option_default(source, "studio_default", "--host") assert ( host_default is not None ), "Could not find --host typer.Option default in studio_default()" - assert host_default == "127.0.0.1", ( - f"studio_default() --host default must be '127.0.0.1' (loopback) " - f"but got '{host_default}'." - ) + assert ( + host_default == "127.0.0.1" + ), f"studio_default() --host default must be '127.0.0.1' (loopback) but got '{host_default}'." def test_studio_run_host_is_loopback(): """`unsloth studio run` --host default must be 127.0.0.1.""" - source = _STUDIO_CMD_PY.read_text() + source = _STUDIO_CMD_PY.read_text(encoding = "utf-8") host_default = _find_typer_option_default(source, "run", "--host") assert host_default is not None, "Could not find --host typer.Option default in run()" assert host_default == "127.0.0.1", ( @@ -71,7 +70,7 @@ def test_studio_run_host_is_loopback(): def test_dns_pinning_opt_out_is_registered_safe_by_default(): - source = _STUDIO_CMD_PY.read_text() + source = _STUDIO_CMD_PY.read_text(encoding = "utf-8") for func_name in ("studio_default", "run"): default = _find_typer_option_default(source, func_name, "--disable-dns-pinning") assert default is False, f"{func_name} must keep DNS pinning enabled by default" diff --git a/tests/studio/test_composer_rtl_bidi_attribute.py b/tests/studio/test_composer_rtl_bidi_attribute.py index defbfab86c..a0afe421e9 100644 --- a/tests/studio/test_composer_rtl_bidi_attribute.py +++ b/tests/studio/test_composer_rtl_bidi_attribute.py @@ -26,22 +26,22 @@ def _block_around( def test_main_composer_has_dir_auto(): # PR #5784 turned the attribute into a JSX conditional; anchor on the inner # "Message input" literal, which survives both spellings. - block = _block_around(THREAD_TSX.read_text(), '"Message input"') + block = _block_around(THREAD_TSX.read_text(encoding = "utf-8"), '"Message input"') assert 'dir="auto"' in block, 'main composer is missing dir="auto"' def test_edit_composer_has_dir_auto(): - block = _block_around(THREAD_TSX.read_text(), "aui-edit-composer-input") + block = _block_around(THREAD_TSX.read_text(encoding = "utf-8"), "aui-edit-composer-input") assert 'dir="auto"' in block, 'edit composer is missing dir="auto"' def test_compare_composer_has_dir_auto(): - block = _block_around(SHARED_TSX.read_text(), "Send to both models") + block = _block_around(SHARED_TSX.read_text(encoding = "utf-8"), "Send to both models") assert 'dir="auto"' in block, 'compare composer is missing dir="auto"' def test_ime_workflow_step_does_not_set_studio_old_pw(): - yml = WORKFLOW_YML.read_text() + yml = WORKFLOW_YML.read_text(encoding = "utf-8") drive_idx = yml.find("Drive IME + multilingual paste regression") assert drive_idx != -1, "IME drive step not found in workflow" next_step_idx = yml.find("- name:", drive_idx + 1) @@ -53,7 +53,7 @@ def test_ime_workflow_step_does_not_set_studio_old_pw(): def test_ime_pass_password_step_does_not_export_old_pw(): - yml = WORKFLOW_YML.read_text() + yml = WORKFLOW_YML.read_text(encoding = "utf-8") pass_idx = yml.find("Pass bootstrap pw for IME / i18n test") assert pass_idx != -1, "IME password setup step not found" next_step_idx = yml.find("- name:", pass_idx + 1) @@ -65,7 +65,7 @@ def test_ime_pass_password_step_does_not_export_old_pw(): def test_ime_playwright_script_does_not_read_studio_old_pw(): - src = IME_PY.read_text() + src = IME_PY.read_text(encoding = "utf-8") code_only = re.sub(r'""".*?"""', "", src, flags = re.DOTALL) assert ( "STUDIO_OLD_PW" not in code_only @@ -76,7 +76,7 @@ def test_ime_playwright_script_does_not_read_studio_old_pw(): def test_main_composer_has_stuck_compositionend_watchdog(): """Issue #5546: WSL Chrome never emits compositionend after IME commit, so the composer needs a watchdog releasing the composing flag or Send stays disabled.""" - src = THREAD_TSX.read_text() + src = THREAD_TSX.read_text(encoding = "utf-8") assert ( "IME_STUCK_TIMEOUT_MS" in src ), "main composer is missing the stuck-compositionend watchdog (issue #5546)" @@ -87,7 +87,7 @@ def test_main_composer_has_stuck_compositionend_watchdog(): def test_compare_composer_has_stuck_compositionend_watchdog(): - src = SHARED_TSX.read_text() + src = SHARED_TSX.read_text(encoding = "utf-8") assert ( "IME_STUCK_TIMEOUT_MS" in src ), "compare composer is missing the stuck-compositionend watchdog (issue #5546)" @@ -97,7 +97,7 @@ def test_compare_composer_has_stuck_compositionend_watchdog(): def test_main_composer_keydown_repins_composing_during_ime(): """Issue #5546: the keydown IME gate must re-pin composingRef so a follow-up Enter does not submit preedit text after the watchdog clears it.""" - src = THREAD_TSX.read_text() + src = THREAD_TSX.read_text(encoding = "utf-8") assert "onKeyDown" in src, "main composer is missing onKeyDown IME gate" assert "e.nativeEvent.isComposing" in src and "keyCode === 229" in src, ( "main composer keydown gate must check both nativeEvent.isComposing " @@ -108,7 +108,7 @@ def test_main_composer_keydown_repins_composing_during_ime(): def test_compare_composer_keydown_repins_composing_during_ime(): """Compare composer onKeyDown re-pins composingRef on IME keypress so a follow-up click-Send during the watchdog window does not slip preedit text.""" - src = SHARED_TSX.read_text() + src = SHARED_TSX.read_text(encoding = "utf-8") assert "composingRef.current = true" in src, ( "compare composer keydown gate must re-pin composingRef when the " "browser still considers the IME active" @@ -142,7 +142,7 @@ def _extract_block( def test_main_composer_keydown_rearms_watchdog(): """After keydown re-pins composingRef the watchdog must re-arm, else the WSL+Chrome no-compositionend path locks Send after any IME keypress (#5546).""" - src = THREAD_TSX.read_text() + src = THREAD_TSX.read_text(encoding = "utf-8") block = _extract_block(src, "const onKeyDown = useCallback") assert "refreshStuckTimer" in block, ( "main composer keydown gate must call refreshStuckTimer after " @@ -159,12 +159,11 @@ def test_main_composer_keydown_rearms_watchdog(): def test_compare_composer_keydown_rearms_watchdog(): """Same re-arm contract for the compare-mode composer.""" - src = SHARED_TSX.read_text() + src = SHARED_TSX.read_text(encoding = "utf-8") block = _extract_block(src, "function onKeyDown", opener = "{", closer = "}") - assert "refreshStuckImeTimer" in block, ( - "compare composer keydown gate must call refreshStuckImeTimer " - "after re-pinning composingRef" - ) + assert ( + "refreshStuckImeTimer" in block + ), "compare composer keydown gate must call refreshStuckImeTimer after re-pinning composingRef" def _assert_enter_guard_before_immediate_recovery(block: str, refresh_call: str) -> None: @@ -177,10 +176,9 @@ def _assert_enter_guard_before_immediate_recovery(block: str, refresh_call: str) "composingRef; candidate-confirming Enter must not submit" ) guard_block = block[enter_idx:recovery_idx] - assert "preventDefault()" in guard_block, ( - "Enter while composingRef is stuck must prevent the same key from " - "falling through to submit" - ) + assert ( + "preventDefault()" in guard_block + ), "Enter while composingRef is stuck must prevent the same key from falling through to submit" assert ( refresh_call in guard_block ), "Enter while composingRef is stuck must keep the watchdog armed" @@ -190,12 +188,12 @@ def _assert_enter_guard_before_immediate_recovery(block: str, refresh_call: str) def test_main_composer_stuck_enter_does_not_clear_before_submit(): - src = THREAD_TSX.read_text() + src = THREAD_TSX.read_text(encoding = "utf-8") block = _extract_block(src, "const onKeyDown = useCallback") _assert_enter_guard_before_immediate_recovery(block, "refreshStuckTimer") def test_compare_composer_stuck_enter_does_not_clear_before_submit(): - src = SHARED_TSX.read_text() + src = SHARED_TSX.read_text(encoding = "utf-8") block = _extract_block(src, "function onKeyDown", opener = "{", closer = "}") _assert_enter_guard_before_immediate_recovery(block, "refreshStuckImeTimer") diff --git a/tests/studio/test_export_output_path_contract.py b/tests/studio/test_export_output_path_contract.py index 8b2f829146..390c569116 100644 --- a/tests/studio/test_export_output_path_contract.py +++ b/tests/studio/test_export_output_path_contract.py @@ -32,7 +32,7 @@ def _return_tuple_arity(fn): def test_export_methods_return_three_tuple_annotation(): - tree = ast.parse(EXPORT.read_text()) + tree = ast.parse(EXPORT.read_text(encoding = "utf-8")) for fn_name in EXPORT_FNS: fn = _find_method(tree, "ExportBackend", fn_name) assert fn is not None, f"missing ExportBackend.{fn_name}" @@ -46,7 +46,7 @@ def test_export_methods_return_three_tuple_annotation(): def test_export_methods_return_three_element_tuples(): - tree = ast.parse(EXPORT.read_text()) + tree = ast.parse(EXPORT.read_text(encoding = "utf-8")) for fn_name in EXPORT_FNS: fn = _find_method(tree, "ExportBackend", fn_name) assert fn is not None @@ -57,7 +57,7 @@ def test_export_methods_return_three_element_tuples(): def test_local_save_assigns_output_path(): - tree = ast.parse(EXPORT.read_text()) + tree = ast.parse(EXPORT.read_text(encoding = "utf-8")) for fn_name in EXPORT_FNS: fn = _find_method(tree, "ExportBackend", fn_name) assert fn is not None @@ -74,7 +74,7 @@ def test_local_save_assigns_output_path(): def test_gpu_save_method_bound_for_hub_only(): - tree = ast.parse(EXPORT.read_text()) + tree = ast.parse(EXPORT.read_text(encoding = "utf-8")) fn = _find_method(tree, "ExportBackend", "export_merged_model") assert fn is not None found_pre_save_method = False @@ -103,7 +103,7 @@ def test_gpu_save_method_bound_for_hub_only(): def test_mlx_hub_only_uses_temp_directory(): - src = EXPORT.read_text() + src = EXPORT.read_text(encoding = "utf-8") assert ( src.count("tempfile.TemporaryDirectory") >= 3 ), "expected TemporaryDirectory in merged, base, and lora hub-push paths" @@ -111,7 +111,7 @@ def test_mlx_hub_only_uses_temp_directory(): def test_is_mlx_imported_from_unsloth(): - src = EXPORT.read_text() + src = EXPORT.read_text(encoding = "utf-8") assert "from unsloth import" in src head = src.split("class ExportBackend")[0] assert "_IS_MLX" in head diff --git a/tests/studio/test_frontend_dep_removal.py b/tests/studio/test_frontend_dep_removal.py index aead44f0cd..ace5955621 100644 --- a/tests/studio/test_frontend_dep_removal.py +++ b/tests/studio/test_frontend_dep_removal.py @@ -53,8 +53,7 @@ CASES: list[Case] = [ ), Case( "C3", - "removing katex is safe: streamdown/math, mermaid, " - "rehype-katex all keep it at top level", + "removing katex is safe: streamdown/math, mermaid, rehype-katex all keep it at top level", ["katex"], "PASS", [], @@ -69,8 +68,7 @@ CASES: list[Case] = [ ), Case( "C6", - "removing @radix-ui/react-slot is safe: pulled by " - "radix-ui umbrella + @assistant-ui/react", + "removing @radix-ui/react-slot is safe: pulled by radix-ui umbrella + @assistant-ui/react", ["@radix-ui/react-slot"], "PASS", [], @@ -852,7 +850,7 @@ ADV_CASES: list[AdvCase] = [ "A12", "JSDoc @import of removed pkg should FAIL", "adv12.ts", - '/** @type {import("__adv_only_pkg_l__").Foo} */\n' "const x = null;\n", + '/** @type {import("__adv_only_pkg_l__").Foo} */\nconst x = null;\n', "__adv_only_pkg_l__", "FAIL", ["__adv_only_pkg_l__"], @@ -1047,7 +1045,7 @@ PKG_FIELD_CASES: list[PkgFieldCase] = [ def run_pkg_field_cases() -> int: - head_pkg = json.loads(HEAD_PKG.read_text()) + head_pkg = json.loads(HEAD_PKG.read_text(encoding = "utf-8")) passed = 0 for pc in PKG_FIELD_CASES: synth_head = json.loads(json.dumps(head_pkg)) @@ -1110,13 +1108,13 @@ def run_pkg_field_cases() -> int: def run_adversarial_cases() -> int: ADVERSARIAL_TMP_DIR.mkdir(parents = True, exist_ok = True) - head_pkg = json.loads(HEAD_PKG.read_text()) + head_pkg = json.loads(HEAD_PKG.read_text(encoding = "utf-8")) passed = 0 for ac in ADV_CASES: # Drop the synthetic file. fpath = ADVERSARIAL_TMP_DIR / ac.filename try: - fpath.write_text(ac.content) + fpath.write_text(ac.content, encoding = "utf-8") # Base adds the target pkg; real head lacks it, so the script # treats it as removed and scans the repo (now with our file). synth_base = json.loads(json.dumps(head_pkg)) @@ -1259,7 +1257,7 @@ ENUM_CASES: list[EnumCase] = [ def run_enum_cases() -> int: - head_pkg = json.loads(HEAD_PKG.read_text()) + head_pkg = json.loads(HEAD_PKG.read_text(encoding = "utf-8")) passed = 0 ADVERSARIAL_TMP_DIR.mkdir(parents = True, exist_ok = True) for ec in ENUM_CASES: @@ -1508,7 +1506,7 @@ def run_wrapper_cases() -> int: def main() -> int: - head_pkg = json.loads(HEAD_PKG.read_text()) + head_pkg = json.loads(HEAD_PKG.read_text(encoding = "utf-8")) print(f"Running {len(CASES)} edge cases against {SCRIPT.relative_to(REPO)}") print() results: list[tuple[Case, bool, str]] = [] diff --git a/tests/studio/test_is_mlx_dispatch_gate.py b/tests/studio/test_is_mlx_dispatch_gate.py index 0e5de1b789..f233b76e4e 100644 --- a/tests/studio/test_is_mlx_dispatch_gate.py +++ b/tests/studio/test_is_mlx_dispatch_gate.py @@ -27,7 +27,7 @@ UNSLOTH_INIT = REPO_ROOT / "unsloth" / "__init__.py" def test_is_mlx_gate_uses_three_required_predicates(): """_IS_MLX must AND Darwin+arm64+importable-mlx; dropping any breaks dispatch.""" - tree = ast.parse(UNSLOTH_INIT.read_text()) + tree = ast.parse(UNSLOTH_INIT.read_text(encoding = "utf-8")) target = None for node in ast.walk(tree): diff --git a/tests/studio/test_llama_cpp_wall_clock_cap.py b/tests/studio/test_llama_cpp_wall_clock_cap.py index b7b6917092..899eb64af7 100644 --- a/tests/studio/test_llama_cpp_wall_clock_cap.py +++ b/tests/studio/test_llama_cpp_wall_clock_cap.py @@ -14,7 +14,7 @@ SOURCE_PATH = ( / "inference" / "llama_cpp.py" ) -SRC = SOURCE_PATH.read_text() +SRC = SOURCE_PATH.read_text(encoding = "utf-8") TREE = ast.parse(SRC) diff --git a/tests/studio/test_mlx_training_worker_behaviors.py b/tests/studio/test_mlx_training_worker_behaviors.py index 78b229d6e9..adffdc501b 100644 --- a/tests/studio/test_mlx_training_worker_behaviors.py +++ b/tests/studio/test_mlx_training_worker_behaviors.py @@ -15,7 +15,7 @@ def _find_func(tree, name): def test_run_mlx_training_passes_token_to_from_pretrained(): - tree = ast.parse(WORKER.read_text()) + tree = ast.parse(WORKER.read_text(encoding = "utf-8")) fn = _find_func(tree, "_run_mlx_training") assert fn is not None found = False @@ -36,7 +36,7 @@ def test_run_mlx_training_passes_token_to_from_pretrained(): def test_wandb_init_strips_secret_keys(): - src = WORKER.read_text() + src = WORKER.read_text(encoding = "utf-8") assert "_wandb_sensitive" in src, "expected a sensitive-key set near wandb.init" assert '"hf_token"' in src and '"wandb_token"' in src assert ( @@ -45,26 +45,26 @@ def test_wandb_init_strips_secret_keys(): def test_local_dataset_loader_uses_load_dataset_path(): - src = WORKER.read_text() + src = WORKER.read_text(encoding = "utf-8") assert "_resolve_mlx_local_dataset_files" in src assert "_mlx_local_dataset_loader_for_files" in src assert "data_files = all_files" in src or "data_files=all_files" in src def test_send_aliases_status_message_to_message(): - src = WORKER.read_text() + src = WORKER.read_text(encoding = "utf-8") assert 'kwargs["message"] = sm' in src or 'kwargs["message"]=sm' in src def test_slice_uses_inclusive_end_and_handles_zero(): - src = WORKER.read_text() + src = WORKER.read_text(encoding = "utf-8") assert "min(end + 1, len(ds))" in src or "min(end+1, len(ds))" in src assert "slice_start if slice_start is not None else 0" in src assert "slice_end if slice_end is not None else len(ds) - 1" in src def test_poll_stop_returns_on_broken_pipe(): - src = WORKER.read_text() + src = WORKER.read_text(encoding = "utf-8") assert "except (EOFError, OSError)" in src lines = src.splitlines() for i, line in enumerate(lines): @@ -83,7 +83,7 @@ def test_poll_stop_returns_on_broken_pipe(): def test_unsloth_zoo_mlx_imports_have_friendly_error(): - src = WORKER.read_text() + src = WORKER.read_text(encoding = "utf-8") assert "from unsloth_zoo.mlx.loader import FastMLXModel" in src assert "from unsloth_zoo.mlx.trainer import" in src assert "raise ImportError" in src diff --git a/tests/studio/test_model_picker_contracts.py b/tests/studio/test_model_picker_contracts.py index 815f68e010..93e0d3e834 100644 --- a/tests/studio/test_model_picker_contracts.py +++ b/tests/studio/test_model_picker_contracts.py @@ -23,7 +23,7 @@ FRONTEND = WORKDIR / "studio" / "frontend" / "src" def _read(rel: str) -> str: path = FRONTEND / rel assert path.exists(), f"missing source file: {path}" - return path.read_text() + return path.read_text(encoding = "utf-8") def test_models_api_sends_token_via_header_not_query(): diff --git a/tests/studio/test_studio_gguf_export_script_pin.py b/tests/studio/test_studio_gguf_export_script_pin.py index defd0d49d4..3d643dc4c3 100644 --- a/tests/studio/test_studio_gguf_export_script_pin.py +++ b/tests/studio/test_studio_gguf_export_script_pin.py @@ -12,7 +12,7 @@ from pathlib import Path SOURCE_PATH = ( Path(__file__).resolve().parents[2] / "studio" / "backend" / "core" / "export" / "export.py" ) -SRC = SOURCE_PATH.read_text() +SRC = SOURCE_PATH.read_text(encoding = "utf-8") TREE = ast.parse(SRC) diff --git a/tests/studio/test_studio_text_descender_clipping.py b/tests/studio/test_studio_text_descender_clipping.py index 98fb3b4b13..c7196a029b 100644 --- a/tests/studio/test_studio_text_descender_clipping.py +++ b/tests/studio/test_studio_text_descender_clipping.py @@ -24,7 +24,7 @@ APP_SIDEBAR = WORKDIR / "studio" / "frontend" / "src" / "components" / "app-side def _read(path: Path) -> str: assert path.exists(), f"missing source file: {path}" - return path.read_text() + return path.read_text(encoding = "utf-8") def test_model_selector_trigger_label_uses_leading_tight(): diff --git a/tests/test_fast_generate_slow_guard.py b/tests/test_fast_generate_slow_guard.py index 6bfc561e54..b32cf3c56c 100644 --- a/tests/test_fast_generate_slow_guard.py +++ b/tests/test_fast_generate_slow_guard.py @@ -13,7 +13,7 @@ UTILS = os.path.join(HERE, "unsloth", "models", "_utils.py") def _load_factory(): - src = open(UTILS).read() + src = open(UTILS, encoding = "utf-8").read() for node in ast.parse(src).body: if isinstance(node, ast.FunctionDef) and node.name == "make_fast_generate_wrapper": ns = {"functools": functools} diff --git a/tests/test_fp8_device_context.py b/tests/test_fp8_device_context.py index 2eea35f4e6..1f72a23ec7 100644 --- a/tests/test_fp8_device_context.py +++ b/tests/test_fp8_device_context.py @@ -78,7 +78,7 @@ class _LaunchVisitor(ast.NodeVisitor): def _load_device_context_helper(fake_torch: _FakeTorch): - source = FP8_SOURCE.read_text() + source = FP8_SOURCE.read_text(encoding = "utf-8") tree = ast.parse(source) for node in tree.body: if isinstance(node, ast.FunctionDef) and node.name == "_fp8_triton_device_context": @@ -144,7 +144,7 @@ def test_fp8_device_context_is_noop_for_non_cuda_tensor() -> None: def test_fp8_triton_launches_enter_tensor_device_context() -> None: - tree = ast.parse(FP8_SOURCE.read_text()) + tree = ast.parse(FP8_SOURCE.read_text(encoding = "utf-8")) function_names = {node.name for node in ast.walk(tree) if isinstance(node, ast.FunctionDef)} assert "_fp8_triton_device_context" in function_names diff --git a/tests/test_gemma4_chat_template.py b/tests/test_gemma4_chat_template.py index cfbc81f736..fa9e253965 100644 --- a/tests/test_gemma4_chat_template.py +++ b/tests/test_gemma4_chat_template.py @@ -14,7 +14,7 @@ CHAT_TEMPLATES_PATH = os.path.join( def _extract_template(name): - src = open(CHAT_TEMPLATES_PATH).read() + src = open(CHAT_TEMPLATES_PATH, encoding = "utf-8").read() pattern = rf'{re.escape(name)}\s*=\s*\\\n"""(.*?)"""' m = re.search(pattern, src, flags = re.DOTALL) assert m, f"Could not extract {name} from chat_templates.py" diff --git a/tests/test_gemma_2b_mapper_key.py b/tests/test_gemma_2b_mapper_key.py index 31edacfce4..5435eb22f8 100644 --- a/tests/test_gemma_2b_mapper_key.py +++ b/tests/test_gemma_2b_mapper_key.py @@ -18,7 +18,7 @@ MAPPER_PATH = os.path.join(os.path.dirname(__file__), os.pardir, "unsloth", "mod def _load_mappers(): - with open(MAPPER_PATH) as f: + with open(MAPPER_PATH, encoding = "utf-8") as f: source = f.read() namespace = {} exec(compile(source, MAPPER_PATH, "exec"), namespace) diff --git a/tests/test_generate_kwarg_gate.py b/tests/test_generate_kwarg_gate.py index 6d1379d3a9..00b3ddf6ee 100644 --- a/tests/test_generate_kwarg_gate.py +++ b/tests/test_generate_kwarg_gate.py @@ -9,7 +9,7 @@ VISION = os.path.join(HERE, "unsloth", "models", "vision.py") def _load_helper(): - src = open(VISION).read() + src = open(VISION, encoding = "utf-8").read() mod = ast.parse(src) for node in mod.body: if isinstance(node, ast.FunctionDef) and node.name == "_unsloth_generate_accepts_kwarg": diff --git a/tests/test_gradient_checkpointing_restore.py b/tests/test_gradient_checkpointing_restore.py index 4f9f3faccc..ee9ef163b6 100644 --- a/tests/test_gradient_checkpointing_restore.py +++ b/tests/test_gradient_checkpointing_restore.py @@ -28,8 +28,8 @@ import re from pathlib import Path _ROOT = Path(__file__).resolve().parent.parent / "unsloth" / "models" -_RL = (_ROOT / "rl.py").read_text() -_RL_REPLACEMENTS = (_ROOT / "rl_replacements.py").read_text() +_RL = (_ROOT / "rl.py").read_text(encoding = "utf-8") +_RL_REPLACEMENTS = (_ROOT / "rl_replacements.py").read_text(encoding = "utf-8") # The single-line ternary form used at the trainer call sites: # ._unsloth_gradient_checkpointing if hasattr(, '...') else getattr(, 'gradient_checkpointing', True) @@ -162,7 +162,7 @@ def test_recording_sites_are_real_module_code(): # string. Assert it's present at the choke point (patch_peft_model, so loaded adapters # are covered) and at the pre-wrapped pass-through, both of which bypass the old # get_peft_model-only recording. - llama = (_ROOT / "llama.py").read_text() + llama = (_ROOT / "llama.py").read_text(encoding = "utf-8") tree = ast.parse(llama) def assigns_marker(node): diff --git a/tests/test_import_fixes_drift.py b/tests/test_import_fixes_drift.py index 0bee68f940..8596bf259d 100644 --- a/tests/test_import_fixes_drift.py +++ b/tests/test_import_fixes_drift.py @@ -704,7 +704,7 @@ def test_accelerate_find_device_skips_empty_logits(): def test_accelerate_patch_wired_into_gpu_init(): """The patch must be installed at startup, not only importable.""" source = Path(__file__).resolve().parent.parent / "unsloth" / "_gpu_init.py" - source = source.read_text() + source = source.read_text(encoding = "utf-8") assert "patch_accelerate_recursively_apply()" in source, ( "DRIFT DETECTED: patch_accelerate_recursively_apply is defined but " "never called in _gpu_init.py, so real imports never install it." diff --git a/tests/test_loader_glob_skip.py b/tests/test_loader_glob_skip.py index ade9e89fde..c37515a8cc 100644 --- a/tests/test_loader_glob_skip.py +++ b/tests/test_loader_glob_skip.py @@ -116,7 +116,7 @@ class TestLoaderSourceHasGuard(unittest.TestCase): loader_path = os.path.join( os.path.dirname(__file__), os.pardir, "unsloth", "models", "loader.py" ) - with open(loader_path) as f: + with open(loader_path, encoding = "utf-8") as f: source = f.read() lines = source.splitlines() diff --git a/tests/test_multi_image_grpo_chunking.py b/tests/test_multi_image_grpo_chunking.py index ea142ce1ef..350dc403cd 100644 --- a/tests/test_multi_image_grpo_chunking.py +++ b/tests/test_multi_image_grpo_chunking.py @@ -12,7 +12,7 @@ SOURCE_PATH = os.path.join(REPO_ROOT, "unsloth", "models", "rl_replacements.py") def _read_source() -> str: - with open(SOURCE_PATH, "r") as fh: + with open(SOURCE_PATH, "r", encoding = "utf-8") as fh: return fh.read() diff --git a/tests/test_offload_embedding_hooks.py b/tests/test_offload_embedding_hooks.py index b8be603b2a..4739372e15 100644 --- a/tests/test_offload_embedding_hooks.py +++ b/tests/test_offload_embedding_hooks.py @@ -11,7 +11,7 @@ VISION = os.path.join(HERE, "unsloth", "models", "vision.py") def _load_installer(): - src = open(VISION).read() + src = open(VISION, encoding = "utf-8").read() mod = ast.parse(src) for node in mod.body: if isinstance(node, ast.FunctionDef) and node.name == "_install_offload_embedding_hooks": diff --git a/tests/test_offload_tied_guard.py b/tests/test_offload_tied_guard.py index 096fba116d..f7f51d6913 100644 --- a/tests/test_offload_tied_guard.py +++ b/tests/test_offload_tied_guard.py @@ -11,7 +11,7 @@ VISION = os.path.join(HERE, "unsloth", "models", "vision.py") def _load_fn(): - src = open(VISION).read() + src = open(VISION, encoding = "utf-8").read() mod = ast.parse(src) for node in mod.body: if isinstance(node, ast.FunctionDef) and node.name == "_embeddings_are_tied": diff --git a/tests/test_source_read_encoding.py b/tests/test_source_read_encoding.py new file mode 100644 index 0000000000..07af605fe0 --- /dev/null +++ b/tests/test_source_read_encoding.py @@ -0,0 +1,1252 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Guard: tests that read checked-in files must name their encoding. + +`Path.read_text()` and `open()` with no encoding use `locale.getencoding()`: +UTF-8 on the Linux and macOS runners, cp1252 on a stock Windows install. A test +that reads a repo file that way passes in CI and raises UnicodeDecodeError for a +Windows contributor as soon as that file gains a non-ASCII byte, which the +source-scanning tests do constantly: + + studio/backend/routes/inference.py carries the DeepSeek tool-call token + regexes, so it holds U+FF5C and U+2581. Reading it as cp1252 dies on + "byte 0x81", taking test_cancel_atomicity.py and test_cancel_id_wiring.py + out at collection time. + +A call is an offence when it does un-pinned text I/O and either of two things +holds. It runs at import, where nothing can see a tmp_path fixture yet. Or the +path it reads anchors on something checked in: a module-level constant or +import, which a fixture parameter can never be, `__file__`, or a relative +literal that names a file actually present in the tree. Anchoring is what +decides the second one, followed through `/` joins, path methods, the locals +and loop variables of the enclosing function, and the parameters of helpers +every caller hands a checked-in path. So `for p in (_B / "routes").rglob("*.py")` +is in scope, `_source(LOADER_PATH)` puts the bare read inside `_source` in +scope, and anything growing out of a tmp_path stays out. That reaches test +bodies, where the same failure lands one step later: + + test_gemma4_chat_template.py opens unsloth/chat_templates.py through a + helper its tests call, and cp1252 cannot decode that file ("byte 0x90"). + test_consent_gate.py reads routes/inference.py as `(_BACKEND / rel)` and + test_gguf_load_cache_reuse.py as `Path(__file__).parent.parent / ...`, both + dying on the same 0x81 the two cancel modules hit at collection. + +Every question the rules ask is answered by the call, its path expression, or +the call sites of the helper it sits in, which keeps them mechanical enough to +enforce with no allowlist and quiet about temp-dir I/O, where the platform +default is harmless and the test wrote the bytes itself. + +Three shapes are consequently out of reach, all fixed by hand and none decidable +from the call. A path a helper hands back rather than takes in, as +`for path in _iter_caller_files()` does in test_security_gate_consistency.py, +says nothing about itself at the read. Text read from a checked-in file and +then written back to a tmp_path, at test_studio_install_workspace_guard.py:851 +and test_scan_packages.py:40, is unsafe only because of where the string came +from. And a read inside a `python -c` snippet, as test_studio_import_no_torch.py +and test_e2e_no_torch_sandbox.py build for their subprocess tests, runs in a +child interpreter this scan never parses: the snippet is an f-string whose paths +are replacement fields, so recovering it would mean evaluating the +interpolation. Reviewers have to catch those three; running the suite under +LC_ALL=C is the cheapest way to find them, since ASCII rejects every byte cp1252 +does and more. +""" + +# `str | None` below is evaluated at import on Python 3.9 without this, and +# pyproject declares requires-python = ">=3.9,<3.15". +from __future__ import annotations + +import ast +import os +import subprocess +from pathlib import Path + +TESTS = Path(__file__).resolve().parent +REPO = TESTS.parent +# Both trees ship to Windows contributors, and separate CI jobs collect them +# (repo-cpu-tests and the studio-backend matrix), so the rule covers both. +# Not a hand-written list: studio/backend/hub/tests and unsloth/kernels/moe/tests +# are already here, and the next one has to be covered the day it lands. +SKIP_DIRS = {".git", ".venv", "build", "dist", "frontend", "node_modules", "site-packages"} + + +def _walked_test_files(repo: Path): + """Every *.py under a tests directory, found by walking.""" + found = [] + for dirpath, dirnames, filenames in os.walk(repo): + dirnames[:] = sorted(d for d in dirnames if d not in SKIP_DIRS) + if "tests" not in Path(dirpath).relative_to(repo).parts: + continue + found.extend(Path(dirpath) / f for f in filenames if f.endswith(".py")) + return found + + +def _tracked_test_files(repo: Path): + """The same, but only what git is actually tracking. + + A walk picks up whatever happens to be lying in the checkout: a scratch + directory, a nested worktree, a vendored dependency. None of those are ours + to police, and a single syntax error in one would fail this test for + everybody who has one. Asking git keeps the promise the docstring makes. + """ + try: + listed = subprocess.run( + ["git", "-C", str(repo), "ls-files", "-z", "--", "*.py"], + capture_output = True, + timeout = 60, + ) + except (OSError, subprocess.SubprocessError): + return None + if listed.returncode != 0: + return None # not a checkout, so fall back to walking + names = listed.stdout.decode("utf-8", errors = "replace").split("\0") + return [ + repo / name + for name in names + if name and "tests" in Path(name).parts and not SKIP_DIRS.intersection(Path(name).parts) + ] + + +SOURCES = _tracked_test_files(REPO) +if SOURCES is None: + SOURCES = _walked_test_files(REPO) +GUARDED_METHODS = {"read_text", "write_text"} +# Openers that are somebody else's are recognised by the file's own imports +# rather than a fixed list, so `import tarfile as tf` and `from PIL import +# Image` are both covered without naming either. +# These wrap their stream in a TextIOWrapper for a "t" mode, which takes the +# platform default exactly like builtin open. Unlike open they default to "rb", +# so only an explicit text mode is in scope. lzma takes encoding keyword-only. +COMPRESSED_OPENERS = {"bz2": 3, "gzip": 3, "lzma": None} +# Wrappers that stay lazy, so draining one drains what it was given. +LAZY_ADAPTERS = {"enumerate", "filter", "islice", "map", "reversed", "zip"} +# Callables that drain a generator argument immediately. +EAGER_CONSUMERS = { + "all", + "any", + "dict", + "frozenset", + "list", + "max", + "min", + "next", + "set", + "sorted", + "sum", + "tuple", +} +# Values that re-select the platform default when passed as the encoding. +PLATFORM_DEFAULT_ENCODINGS = (None, "locale") +# `Path.read_text(p)` is the unbound spelling of `p.read_text()`: same API, same +# platform default, but the instance takes the first slot so every argument +# shifts one place right. +PATH_CLASSES = {"Path", "PosixPath", "PurePath", "WindowsPath"} +# Modules whose `open` IS the builtin: same signature, same platform default. +BUILTIN_OPEN_MODULES = {"builtins", "io"} +# Receivers `self.SOURCE` and `cls.SOURCE` reach a class attribute through. +SELF_NAMES = {"cls", "self"} +# A module-level name is normally an anchor, since a fixture cannot reach one. +# These build a directory the run owns, so a name rooted in one is temp I/O +# however it is spelled, and the platform default there is harmless. +TEMP_FACTORIES = { + "NamedTemporaryFile", + "TemporaryDirectory", + "gettempdir", + "mkdtemp", + "mkstemp", +} +# Functions that hand back a path still pointing at their first argument. An +# unlisted call is left unresolved: a helper may well return a temp copy of what +# it was given, and following it would put test-created files back in scope. +PATH_FUNCTIONS = { + "abspath", + "dirname", + "expanduser", + "fspath", + "join", + "normpath", + "realpath", + "relpath", + "str", +} +# Path methods that hand back another path, so the receiver is still the anchor. +PATH_METHODS = { + "absolute", + "as_posix", + "expanduser", + "glob", + "iterdir", + "joinpath", + "resolve", + "rglob", + "with_name", + "with_stem", + "with_suffix", +} +# Where each API takes its encoding positionally, for the bound call. +ENCODING_POSITION = {"read_text": 0, "write_text": 1, "Path.open": 2, "open": 3} +# Distinct from None so that "no mode argument at all" still means text. +UNKNOWN_MODE = object() +# Stand-in for a file whose imports are not to hand, so every helper can be +# called on its own without pretending it knows what was imported. +NO_MODULES: dict = {} + + +def _static_truth(node: ast.AST): + """Whether a condition is a literal true or false, else None for "depends".""" + return bool(node.value) if isinstance(node, ast.Constant) else None + + +def _live_branches(node: ast.AST): + """The children of a branch that can actually run, or None if it is not one. + + `if False:` and the right of `False and ...` never execute, so reporting a + read there is a CI failure with no reachable cause and no correct edit. + """ + if isinstance(node, ast.If): + taken = _static_truth(node.test) + if taken is None: + return None + return [node.test, *(node.body if taken else node.orelse)] + if isinstance(node, ast.IfExp): + taken = _static_truth(node.test) + if taken is None: + return None + return [node.test, node.body if taken else node.orelse] + if isinstance(node, ast.BoolOp) and node.values: + # `and` stops at the first false operand, `or` at the first true one. + stops = isinstance(node.op, ast.Or) + live = [] + for value in node.values: + live.append(value) + if _static_truth(value) is stops: + break + return live if len(live) < len(node.values) else None + return None + + +def _callee_name(func: ast.AST): + """The bare name a callee ends in, whether or not it is qualified.""" + return func.id if isinstance(func, ast.Name) else getattr(func, "attr", None) + + +def _is_main_guard(node: ast.AST) -> bool: + """True for `if __name__ == "__main__":`, whose body never runs at import. + + The operator has to be `==`: `if __name__ != "__main__":` runs its body at + import, so treating it as script-only would invert the rule. + """ + if not isinstance(node, ast.If) or not isinstance(node.test, ast.Compare): + return False + if not all(isinstance(op, ast.Eq) for op in node.test.ops): + return False + operands = [node.test.left, *node.test.comparators] + # Either spelling: `__name__ == "__main__"` or `"__main__" == __name__`. + has_name = any(isinstance(o, ast.Name) and o.id == "__name__" for o in operands) + has_main = any(isinstance(o, ast.Constant) and o.value == "__main__" for o in operands) + return has_name and has_main + + +def _is_eager_consumer(func: ast.expr) -> bool: + """True for a callee that drains a generator argument on the spot. + + iter/zip/map/filter/enumerate/reversed hand back another lazy object, so a + genexp passed to those still has not run. + """ + if isinstance(func, ast.Attribute): + return func.attr in {"join", "extend", "update", "writelines"} + return isinstance(func, ast.Name) and func.id in EAGER_CONSUMERS + + +def _import_time_calls(tree: ast.Module): + """Yield Call nodes that run at import time. + + That is module scope, class bodies, and the bodies of module-level helpers + invoked from either. A helper is the same hazard as an inline read: + `CODE = _extract_mixed_precision_code()` runs its `read_text()` during + collection, so skipping every def would let the Windows failure back in. + + A def's body waits for a call, but its decorators and argument defaults run + when the def executes, so those are followed. Lambda bodies are skipped for + the same reason, as is everything but the outermost iterable of a generator + expression. List, set and dict comprehensions are walked in full: unlike a + genexp they run their element, filters and nested iterators immediately. + + A body is only ever entered through an executed statement, never by walking + into a def, so the "this definitely runs" property that makes the rule + allowlist-free holds. Not followed: the body of + `if __name__ == "__main__":`, which pytest never runs (its `else` arm does, + so that is walked), and non-name calls, which are left unresolved rather + than guessed at. + """ + # Defs reachable from a scope that executes at import: module body, any + # class body, and (added when the helper is entered) any def nested inside + # a helper we follow. `class F: def _load(): ...; DATA = _load()` runs + # _load while the class is constructed. + helpers: dict = {} + + def _collect(body): + scopes = [body] + while scopes: + for node in scopes.pop(): + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + helpers.setdefault(node.name, node) + elif isinstance(node, ast.ClassDef): + scopes.append(node.body) + + _collect(tree.body) + consumed = _eagerly_consumed(tree) + entered = set() + frontier = [list(tree.body)] + while frontier: + stack = frontier.pop() + while stack: + node = stack.pop() + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + # The body waits for a call; these two run right now. + stack.extend(node.decorator_list) + stack.extend(d for d in node.args.defaults if d is not None) + stack.extend(d for d in node.args.kw_defaults if d is not None) + continue + if isinstance(node, ast.Lambda): + stack.extend(d for d in node.args.defaults if d is not None) + stack.extend(d for d in node.args.kw_defaults if d is not None) + continue + if isinstance(node, ast.GeneratorExp) and id(node) not in consumed: + # Lazy: only the outermost iterable is evaluated where written. + if node.generators: + stack.append(node.generators[0].iter) + continue + if _is_main_guard(node): + stack.extend(node.orelse) # the else arm runs at import + continue + live = _live_branches(node) + if live is not None: + stack.extend(live) # the dead arm never runs, so nothing in it does + continue + if isinstance(node, ast.Call): + yield node + func = node.func + if isinstance(func, ast.Name) and func.id in helpers and func.id not in entered: + helper = helpers[func.id] + # `READS = _load(paths)` on a generator function only builds + # the generator, so its body waits for a consumer just as a + # genexp does. + if not _is_generator(helper) or id(node) in consumed: + entered.add(func.id) + body = list(helper.body) + _collect(body) # a def nested here is now callable + frontier.append(body) + stack.extend(ast.iter_child_nodes(node)) + + +def _eagerly_consumed(tree: ast.Module) -> set: + """Nodes whose lazy value is drained right where it is written. + + Covers both things that defer: a generator expression, and a call to a + generator function. Neither runs its body until something pulls from it, so + an unconsumed one has not happened yet. + """ + # `texts = (p.read_text() for p in ...)` then `list(texts)` consumes the + # generator through a name, so the name has to lead back to it. + named: dict = {} + for node in ast.walk(tree): + if isinstance(node, ast.Assign) and len(node.targets) == 1: + target = node.targets[0] + if isinstance(target, ast.Name) and isinstance(node.value, ast.GeneratorExp): + named.setdefault(target.id, node.value) + + def _resolve(node): + if isinstance(node, ast.Name) and node.id in named: + return named[node.id] + return node + + consumed = set() + for node in ast.walk(tree): + if isinstance(node, ast.Call) and _is_eager_consumer(node.func): + consumed.update(id(_resolve(a)) for a in node.args) + consumed.update(id(_resolve(k.value)) for k in node.keywords) + elif isinstance(node, (ast.For, ast.AsyncFor, ast.comprehension)): + consumed.add(id(_resolve(node.iter))) # the loop pulls every item + # `list(enumerate(_paths()))` drains _paths() as well, one wrapper down. + by_id = {id(n): n for n in ast.walk(tree)} + queue = [by_id[i] for i in list(consumed) if i in by_id] + while queue: + node = queue.pop() + if isinstance(node, ast.Call) and _callee_name(node.func) in LAZY_ADAPTERS: + for arg in node.args: + target = _resolve(arg) + if id(target) not in consumed: + consumed.add(id(target)) + queue.append(target) + return consumed + + +def _temp_rooted_names(tree: ast.Module) -> set: + """Module-level names anchored on a directory the run itself created.""" + names = set() + for node in tree.body: + value = node.value if isinstance(node, (ast.Assign, ast.AnnAssign)) else None + if value is None: + continue + if any( + isinstance(n, ast.Call) and _callee_name(n.func) in TEMP_FACTORIES + for n in ast.walk(value) + ): + targets = node.targets if isinstance(node, ast.Assign) else [node.target] + names.update(t.id for t in targets if isinstance(t, ast.Name)) + return names + + +def _non_path_names(tree: ast.Module) -> set: + """Module-level names bound to a call that plainly does not make a path. + + `response = requests.get(...)` then `response.read_text()` at import is not + pathlib I/O, and demanding an encoding there leaves no compliant edit. + """ + names = set() + for node in tree.body: + if not isinstance(node, ast.Assign) or not isinstance(node.value, ast.Call): + continue + func = node.value.func + if _is_path_preserving(func) or _callee_name(func) in PATH_METHODS: + continue + names.update(t.id for t in node.targets if isinstance(t, ast.Name)) + return names + + +def _is_generator(func) -> bool: + """True when calling this only builds a generator, leaving the body unrun. + + Yields inside a nested def belong to that def, so those do not count. + """ + stack = list(func.body) + while stack: + node = stack.pop() + if isinstance(node, (ast.Yield, ast.YieldFrom)): + return True + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)): + continue + stack.extend(ast.iter_child_nodes(node)) + return False + + +def _module_level_names(tree: ast.Module) -> set: + """Names assigned at module scope.""" + + def _bound(target): + # `SOURCE, CONFIG = Path(...), Path(...)` binds both. + if isinstance(target, ast.Name): + yield target.id + elif isinstance(target, (ast.Tuple, ast.List)): + for element in target.elts: + yield from _bound(element) + elif isinstance(target, ast.Starred): + yield from _bound(target.value) + + def _is_temp(value) -> bool: + return value is not None and any( + isinstance(n, ast.Call) and _callee_name(n.func) in TEMP_FACTORIES + for n in ast.walk(value) + ) + + names = set() + for node in tree.body: + if isinstance(node, ast.Assign): + if _is_temp(node.value): + continue # TMP = Path(tempfile.mkdtemp()) is not checked in + for target in node.targets: + names.update(_bound(target)) + elif isinstance(node, ast.AnnAssign): + if _is_temp(node.value): + continue + names.update(_bound(node.target)) + elif isinstance(node, (ast.Import, ast.ImportFrom)): + # `start._CODEX_FALLBACK_PROMPT` is a path another module defines at + # its own module scope, so the import is an anchor like any constant. + names.update((a.asname or a.name).split(".")[0] for a in node.names) + return names + + +def _local_names(func) -> set: + """Every name the function binds, so a module constant it shadows is skipped. + + Walking nested defs too over-approximates, which only ever drops a call from + the scan. + """ + args = func.args + names = {a.arg for a in [*args.posonlyargs, *args.args, *args.kwonlyargs]} + for extra in (args.vararg, args.kwarg): + if extra is not None: + names.add(extra.arg) + stack = list(ast.iter_child_nodes(func)) + while stack: + node = stack.pop() + if isinstance(node, (ast.ListComp, ast.SetComp, ast.DictComp, ast.GeneratorExp)): + # A comprehension target binds in its own scope, so it shadows + # nothing out here; the rest of the comprehension still does. + for gen in node.generators: + stack.append(gen.iter) + stack.extend(gen.ifs) + stack.extend([node.key, node.value] if isinstance(node, ast.DictComp) else [node.elt]) + continue + if isinstance(node, ast.Name) and isinstance(node.ctx, (ast.Store, ast.Del)): + names.add(node.id) + elif isinstance(node, (ast.Import, ast.ImportFrom)): + names.update((a.asname or a.name).split(".")[0] for a in node.names) + stack.extend(ast.iter_child_nodes(node)) + return names + + +def _imported_names(node) -> dict: + """Names this scope's own imports bind, mapped to where they came from. + + The name alone is not enough in either direction. `import gzip as gz` binds + a name nobody would recognise to an opener that does take an encoding, and + `from PIL.Image import open` binds a name everybody recognises to one that + does not. Keeping the origin settles both. + + Nested function bodies are left out: an import inside one is that + function's business, and treating it as the module's would let a single + local `from PIL.Image import open` turn off the builtin check everywhere. + """ + bound = {} + stack = list(ast.iter_child_nodes(node)) + while stack: + item = stack.pop() + if isinstance(item, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)): + continue + if isinstance(item, (ast.Import, ast.ImportFrom)): + bound.update(_import_bindings(item)) + else: + stack.extend(ast.iter_child_nodes(item)) + return bound + + +def _import_bindings(node) -> dict: + """What one import statement binds, mapped to where each name came from.""" + if isinstance(node, ast.Import): + return {(a.asname or a.name).split(".")[0]: a.name for a in node.names} + return { + a.asname or a.name: (f"{node.module}.{a.name}" if node.module else a.name) + for a in node.names + } + + +def _imports_at_each_call(tree: ast.Module) -> dict: + """The imports visible at every call, keyed by node id. + + A function's own imports are added on the way in and go out of view again + on the way out, which is what keeps a local alias local. Within a scope they + accumulate in statement order, so `DATA = open(p)` above a later + `from gzip import open` still resolves to the builtin it actually called. + """ + visible_at = {} + + def walk(node, visible): + if isinstance(node, ast.Call): + visible_at[id(node)] = dict(visible) + if isinstance(node, (ast.Import, ast.ImportFrom)): + visible.update(_import_bindings(node)) + return + if isinstance(node, ast.If): + # Only a branch that certainly runs may bind a name for the code + # after it; the others are explored with a copy that is thrown away. + taken = _static_truth(node.test) + walk(node.test, visible) + for arm, runs in ((node.body, taken is not False), (node.orelse, taken is not True)): + if not runs: + continue + inner = visible if taken is not None else dict(visible) + for child in arm: + walk(child, inner) + return + for child in ast.iter_child_nodes(node): + if isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)): + walk(child, dict(visible)) # its own scope, so its own copy + else: + walk(child, visible) + + walk(tree, {}) + return visible_at + + +def _open_alias(name, modules): + """What a bare callable resolves to: "builtin", a COMPRESSED_OPENERS key, or None. + + `from io import open as io_open` is the builtin under another name and + `from gzip import open as gzopen` is gzip's, while `from PIL.Image import + open` is neither and takes no encoding at all. + """ + origin = modules.get(name) + if origin is None: + return "builtin" if name == "open" else None + parts = origin.split(".") + if parts[-1] != "open": + return None + if parts[0] in BUILTIN_OPEN_MODULES or origin == "open": + return "builtin" + return parts[0] if parts[0] in COMPRESSED_OPENERS else None + + +def _origin_root(name, modules) -> str: + """The top-level module a bound name came from, or the name itself.""" + return modules.get(name, name).split(".")[0] + + +def _compressed_key(name, modules): + """The COMPRESSED_OPENERS entry this receiver resolves to, if any.""" + for candidate in (name, _origin_root(name, modules)): + if candidate in COMPRESSED_OPENERS: + return candidate + return None + + +def _is_path_class(name, modules) -> bool: + """True for a pathlib class, including under an alias. + + `from pathlib import Path as P` still puts the instance in slot 0 of an + unbound `P.read_text(SOURCE)`, so matching the bare name is not enough. + """ + if name is None: + return False + return (modules.get(name) or name).split(".")[-1] in PATH_CLASSES + + +def _is_path_attr(node: ast.AST) -> bool: + """True for a qualified path class, as in `pathlib.Path` or `pl.Path`.""" + return isinstance(node, ast.Attribute) and node.attr in PATH_CLASSES + + +def _is_path_preserving(func) -> bool: + """True for a call whose result still points at its first argument. + + Qualified spellings count: `pathlib.Path(p)` and `os.path.join(p, x)` are + the same constructors as the bare names. + """ + name = _callee_name(func) + return name in PATH_CLASSES or name in PATH_FUNCTIONS + + +def _is_module_receiver(name, modules) -> bool: + """True for a receiver that is not itself a path.""" + return ( + name in modules + or _is_path_class(name, modules) + or _compressed_key(name, modules) is not None + or _origin_root(name, modules) in BUILTIN_OPEN_MODULES + ) + + +def _path_expr(call: ast.Call, modules = NO_MODULES): + """The expression naming the file the call reads. + + Usually the receiver, but a module or the Path class in that slot means the + path is the first argument instead: `Path.read_text(REPO / "x.py")` and + `gzip.open(path, "rt")` both read their argument, not `Path` or `gzip`. + """ + func = call.func + if isinstance(func, ast.Attribute): + if _is_path_attr(func.value) or ( + isinstance(func.value, ast.Name) and _is_module_receiver(func.value.id, modules) + ): + return call.args[0] if call.args else _path_keyword(call) + return func.value + if isinstance(func, ast.Name) and _open_alias(func.id, modules) is not None: + return call.args[0] if call.args else _path_keyword(call) + return None + + +def _path_keyword(call: ast.Call): + """The path passed by keyword: `file` for open, `filename` for gzip and kin.""" + for kw in call.keywords: + if kw.arg in ("file", "filename"): + return kw.value + return None + + +def _path_root(node: ast.AST) -> ast.AST: + """Follow a path expression back to whatever it is anchored on. + + `(_BACKEND / rel).read_text()` anchors on _BACKEND and + `Path(__file__).parent / "routes"` on __file__, so joining a relative name + onto a checked-in root stays in scope. Anchoring is what decides it, not the + names further down: `tmp_path / SUBDIR` anchors on the fixture, so a + constant used as a leaf cannot drag temp-dir I/O in. + """ + while True: + if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Div): + node = node.left + elif isinstance(node, (ast.Attribute, ast.Subscript)): + if ( + isinstance(node, ast.Attribute) + and isinstance(node.value, ast.Name) + and node.value.id in SELF_NAMES + ): + return node # self.SOURCE names the class attribute, not self + node = node.value + elif isinstance(node, ast.Call): + func = node.func + # `p.rglob("*.py")` anchors on p, not on the pattern, while + # Path(x), str(x) and os.path.join(x, ...) anchor on the argument. + if isinstance(func, ast.Attribute) and func.attr in PATH_METHODS: + node = func.value + elif _is_path_preserving(func) and node.args: + node = node.args[0] + else: + # An unrecognised call says nothing about where its result + # points, so tempfile.mkdtemp() and a helper that copies its + # argument into a temp dir both stop here. + return node + else: + return node + + +def _is_checked_in_root( + node: ast.AST, + module_names: set, + shadowed, + derived = (), + attrs = (), +) -> bool: + """True when a path expression anchors on something that ships in the repo.""" + if isinstance(node, (ast.Tuple, ast.List, ast.Set)): + # `for path in (MODEL_SELECTOR, APP_SIDEBAR)` is checked in when every + # element is, which is what makes the loop variable one too. + return bool(node.elts) and all( + _is_checked_in_root( + e.value if isinstance(e, ast.Starred) else e, + module_names, + shadowed, + derived, + attrs, + ) + for e in node.elts + ) + root = _path_root(node) + if isinstance(root, ast.Constant) and isinstance(root.value, str): + # A relative literal naming something that exists here is checked in; a + # path the test creates at runtime is not in the tree to be found. + value = root.value + if not value or "\n" in value or "\0" in value or os.path.isabs(value): + return False + try: + return (REPO / value).exists() + except OSError: + return False # too long to be a name, so not one + if isinstance(root, ast.Attribute): + # `self.SOURCE`, where the class body bound SOURCE to a checked-in path. + return root.attr in attrs + if not isinstance(root, ast.Name): + return False + if root.id in derived: + return True + return root.id == "__file__" or (root.id in module_names and root.id not in shadowed) + + +def _class_path_attrs(tree: ast.Module, module_names: set) -> set: + """Class-body names bound to a checked-in path, read back as `self.NAME`. + + `class T: _SETUP_SH = ROOT / "setup.sh"` then `self._SETUP_SH.read_text()` + is as statically provable as the module-level spelling, and the repository + reads seven real source files exactly that way. + """ + attrs, mixed = set(), set() + for node in ast.walk(tree): + if not isinstance(node, ast.ClassDef): + continue + for stmt in node.body: + if isinstance(stmt, ast.Assign): + targets = stmt.targets + elif isinstance(stmt, ast.AnnAssign) and stmt.value is not None: + targets = [stmt.target] + else: + continue + bound = {t.id for t in targets if isinstance(t, ast.Name)} + # One attribute name, two classes, two meanings: only one of them is + # provable, so neither is claimed. Same rule as the local walk. + found = attrs if _is_checked_in_root(stmt.value, module_names, ()) else mixed + found.update(bound) + return attrs - mixed + + +def _reads_itself(name: str, value: ast.AST) -> bool: + """`source = source.read_text()` reads the path before replacing it. + + The name holds a checked-in path right up to that call, so the assignment + is not evidence against it; it is the very read we are looking for. + """ + if not isinstance(value, ast.Call): + return False + expr = _path_expr(value) + return isinstance(expr, ast.Name) and expr.id == name + + +def _unpack(target, value, paired: bool): + """Yield (name node, the value it is bound to) for one binding. + + A destructured target contributes every name inside it. Where the two sides + line up, as in `A, B = P1, P2`, each name takes its own element; where they + do not, as in `for name, path in CASES`, they all take the iterable, which + is the thing whose provenance is known. + """ + if isinstance(target, ast.Name): + yield target, value + return + if not isinstance(target, (ast.Tuple, ast.List)): + return + elements = None + if paired and isinstance(value, (ast.Tuple, ast.List)) and len(value.elts) == len(target.elts): + elements = value.elts + for index, element in enumerate(target.elts): + if isinstance(element, ast.Starred): + element = element.value + yield from _unpack(element, elements[index] if elements else value, paired) + + +def _checked_in_locals( + func, + module_names: set, + shadowed, + seed = (), +) -> set: + """Locals that only ever hold a checked-in path. + + `route = Path(_BACKEND_DIR) / "routes" / "inference.py"` followed by + `route.read_text()` is the same read one line apart. A name bound any other + way, or assigned anything else anywhere in the scope, is not tracked, and + the pass repeats so that a path built up over several locals still counts. + """ + assignments = [] + targets = set() + bad = set() + for node in ast.walk(func): + paired = False + if isinstance(node, ast.Assign) and len(node.targets) == 1: + target, value = node.targets[0], node.value + paired = True # `A, B = P1, P2` lines its sides up element by element + elif isinstance(node, (ast.For, ast.AsyncFor, ast.comprehension)): + # `for p in SRC_DIR.rglob("*.py")` binds p to a checked-in path too, + # and `for name, path in CASES` binds both to the same iterable. + target, value = node.target, node.iter + else: + continue + for name_node, bound in _unpack(target, value, paired): + targets.add(id(name_node)) + if not _reads_itself(name_node.id, bound): + assignments.append((name_node.id, bound)) + for node in ast.walk(func): + # A with-as or an augassign says nothing about the value it binds. + if isinstance(node, ast.Name) and isinstance(node.ctx, (ast.Store, ast.Del)): + if id(node) not in targets: + bad.add(node.id) + args = func.args + bad.update(a.arg for a in [*args.posonlyargs, *args.args, *args.kwonlyargs]) + # A parameter every caller hands a checked-in path is the exception. + bad -= set(seed) + good: set = set(seed) + while True: + grown = set(good) | { + name + for name, value in assignments + if name not in bad and _is_checked_in_root(value, module_names, shadowed, good) + } + # A name assigned a checked-in path somewhere and something else + # elsewhere stays out, since only one of the two is provable. + grown -= { + name + for name, value in assignments + if name in grown and not _is_checked_in_root(value, module_names, shadowed, good) + } + if grown == good: + return good + good = grown + + +def _unwrap_param(node: ast.AST) -> ast.AST: + """`pytest.param(SOURCE, id = "x")` is a wrapper around the real value.""" + if ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) + and node.func.attr == "param" + and node.args + ): + return node.args[0] + return node + + +def _parametrized_values(func) -> dict: + """Parameter values supplied by @pytest.mark.parametrize. + + pytest calls a parametrized test itself, so the decorator is the only call + site there is; without reading it every such parameter looks unprovable. + """ + supplied: dict = {} + for decorator in func.decorator_list: + if not isinstance(decorator, ast.Call) or len(decorator.args) < 2: + continue + if not isinstance(decorator.func, ast.Attribute) or decorator.func.attr != "parametrize": + continue + names, values = decorator.args[0], decorator.args[1] + if not isinstance(names, ast.Constant) or not isinstance(names.value, str): + continue + if not isinstance(values, (ast.List, ast.Tuple, ast.Set)): + continue + argnames = [n.strip() for n in names.value.split(",") if n.strip()] + for element in values.elts: + paired = len(argnames) > 1 and isinstance(element, (ast.Tuple, ast.List)) + row = element.elts if paired else [element] + for argname, value in zip(argnames, row): + supplied.setdefault(argname, []).append(_unwrap_param(value)) + return supplied + + +def _checked_in_params(tree: ast.Module, module_names: set) -> set: + """(function, parameter) pairs that only ever receive a checked-in path. + + `_source(LOADER_PATH)` is what tells us that the `path` parameter of + `_source` is reading a file that ships in the repo; the bare + `path.read_text()` inside it cannot say so on its own. One hop only, and a + parameter any call leaves out, or passes anything else, is not tracked. + + Definitions are held by identity, not by name. Two tests that each nest a + `_read` helper are two different functions, and merging them would let the + one handed a tmp_path rule out what the other proves. + """ + # Every definition, plus which scope it was written in, so a call resolves + # to the nearest enclosing `def` of that name the way Python resolves it. + scope_of: dict = {} + defs_in: dict = {} + + def _index(node, scope): + for child in ast.iter_child_nodes(node): + if isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef)): + defs_in.setdefault(id(scope), {}).setdefault(child.name, child) + scope_of[id(child)] = scope + _index(child, child) + elif isinstance(child, ast.ClassDef): + _index(child, scope) # a class body is not a name lookup scope + else: + _index(child, scope) + + _index(tree, tree) + + def _lookup(name, scope): + while scope is not None: + found = defs_in.get(id(scope), {}).get(name) + if found is not None: + return found + scope = scope_of.get(id(scope)) + return None + + # Which function each call sits in, so a parameter already known to hold a + # checked-in path can be passed on to the next helper. + owner: dict = {} + + def _mark(node, owning): + if isinstance(node, ast.Call): + owner[id(node)] = owning + for child in ast.iter_child_nodes(node): + nested = isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef)) + _mark(child, child if nested else owning) + + _mark(tree, None) + # Which class body each call sits in, so `self._read(...)` resolves to that + # class's method and not a same-named one in a sibling class. + in_class: dict = {} + + def _mark_class(node, cls): + if isinstance(node, ast.Call): + in_class[id(node)] = cls + for child in ast.iter_child_nodes(node): + _mark_class(child, child if isinstance(child, ast.ClassDef) else cls) + + _mark_class(tree, None) + + def _method(cls, name): + if cls is None: + return None + for stmt in cls.body: + if isinstance(stmt, (ast.FunctionDef, ast.AsyncFunctionDef)) and stmt.name == name: + return stmt + return None + + good: set = set() + while True: + grown, bad = set(), set() + for fnode in [d for scope in defs_in.values() for d in scope.values()]: + for argname, values in _parametrized_values(fnode).items(): + ok = all(_is_checked_in_root(v, module_names, ()) for v in values) + (grown if ok else bad).add((id(fnode), argname)) + # What the calling function itself can prove, recomputed each pass so a + # parameter resolved last time can feed a local this time. + scope: dict = {} + for call in ast.walk(tree): + if not isinstance(call, ast.Call): + continue + caller = owner.get(id(call)) + callee, bound = call.func, False + if isinstance(callee, ast.Name): + func = _lookup(callee.id, caller if caller is not None else tree) + elif ( + isinstance(callee, ast.Attribute) + and isinstance(callee.value, ast.Name) + and callee.value.id in SELF_NAMES + ): + # `self._read(ROOT / "x.py")` seeds `_read`'s path parameter too. + func, bound = _method(in_class.get(id(call)), callee.attr), True + else: + continue + if func is None or any(isinstance(a, ast.Starred) for a in call.args): + continue + if caller is None: + here = set() + elif id(caller) in scope: + here = scope[id(caller)] + else: + params = {p for f, p in good if f == id(caller)} + here = _checked_in_locals(caller, module_names, _local_names(caller), params) + scope[id(caller)] = here + positional = [a.arg for a in [*func.args.posonlyargs, *func.args.args]] + if bound: + positional = positional[1:] # the receiver already fills `self` + # A keyword-only parameter never takes a positional slot, so it is + # matched by name alone. + params = positional + [a.arg for a in func.args.kwonlyargs] + supplied = dict(zip(positional, call.args)) + supplied.update({k.arg: k.value for k in call.keywords if k.arg in params}) + for param in params: + value = supplied.get(param) + ok = value is not None and _is_checked_in_root(value, module_names, (), here) + (grown if ok else bad).add((id(func), param)) + grown -= bad + if grown == good: + return good + good = grown + + +def _checked_in_path_calls( + tree: ast.Module, + modules = NO_MODULES, + visible_at = None, +): + """Yield calls, at any depth, whose path is provably a checked-in file. + + The import-time walk alone leaves test bodies unguarded, and a bare read + there is the same Windows failure one step later: `_extract_template()` in + test_gemma4_chat_template.py opens unsloth/chat_templates.py, which cp1252 + cannot decode ("byte 0x90"), so the test errors rather than the collection. + + Two spellings qualify. A tmp_path arrives as a fixture parameter and a + tempfile is built in the body, so neither can be bound at module scope nor + derived from `__file__`. That keeps temp-dir I/O out of scope without an + allowlist, since there the platform default is harmless and the test wrote + the bytes itself. + """ + module_names = _module_level_names(tree) + consumed = _eagerly_consumed(tree) + visible_at = _imports_at_each_call(tree) if visible_at is None else visible_at + params = _checked_in_params(tree, module_names) + attrs = _class_path_attrs(tree, module_names) + + def visit( + node, + shadowed, + derived = frozenset(), + ): + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)): + shadowed = shadowed | _local_names(node) + # Seed with the parameters first: `p = root / "x.py"` is only + # derivable once `root` is known to hold a checked-in path. + seeded = {p for f, p in params if f == id(node)} + derived = _checked_in_locals(node, module_names, shadowed, seeded) + elif _is_main_guard(node): + # Never runs under pytest, so rule 1 skips it for the same reason. + for child in node.orelse: + yield from visit(child, shadowed, derived) + return + elif isinstance(node, ast.GeneratorExp) and id(node) not in consumed: + if node.generators: + yield from visit(node.generators[0].iter, shadowed, derived) + return + elif (live := _live_branches(node)) is not None: + for child in live: + yield from visit(child, shadowed, derived) + return + elif isinstance(node, ast.Call): + expr = _path_expr(node, visible_at.get(id(node), modules)) + if expr is not None and _is_checked_in_root( + expr, module_names, shadowed, derived, attrs + ): + yield node + for child in ast.iter_child_nodes(node): + yield from visit(child, shadowed, derived) + + yield from visit(tree, frozenset()) + + +def _open_mode(call: ast.Call, mode_index: int): + """The literal mode of an open() call, or UNKNOWN_MODE. + + A splat or a non-literal hides the mode. Defaulting those to "r" would + demand an encoding on a call that may resolve to "rb", where passing one is + a ValueError, so the contributor would have no compliant edit. + """ + if any(isinstance(a, ast.Starred) for a in call.args): + return UNKNOWN_MODE + if any(kw.arg is None for kw in call.keywords): + return UNKNOWN_MODE + if len(call.args) > mode_index: + node = call.args[mode_index] + return node.value if isinstance(node, ast.Constant) else UNKNOWN_MODE + for kw in call.keywords: + if kw.arg == "mode": + return kw.value.value if isinstance(kw.value, ast.Constant) else UNKNOWN_MODE + return "r" + + +def _is_text(call: ast.Call, mode_index: int) -> bool: + mode = _open_mode(call, mode_index) + return mode is not UNKNOWN_MODE and "b" not in str(mode) + + +def _names_encoding(call: ast.Call) -> bool: + """True only for an encoding that actually pins one. + + `encoding = None` and `encoding = "locale"` both re-select the platform + default, so the keyword being present is not enough. A `**kwargs` may carry + one we cannot see, so it counts as named rather than risking a false alarm. + """ + for kw in call.keywords: + if kw.arg is None: + return True + if kw.arg != "encoding": + continue + if isinstance(kw.value, ast.Constant) and kw.value.value in PLATFORM_DEFAULT_ENCODINGS: + return False + return True + return False + + +def _pins_encoding(call: ast.Call, position: int | None) -> bool: + """True when the call names an encoding, positionally or by keyword. + + `position` is None where the API takes it keyword-only. A splat makes the + positions meaningless, so it counts as named rather than demanding an edit + the contributor cannot make correctly. + """ + if any(isinstance(a, ast.Starred) for a in call.args): + return True + if position is not None and len(call.args) > position: + node = call.args[position] + if isinstance(node, ast.Constant): + return node.value not in PLATFORM_DEFAULT_ENCODINGS + return True + return _names_encoding(call) + + +def _offender(call: ast.Call, modules = NO_MODULES) -> str | None: + """The call's name if it reads text without an encoding, else None.""" + func = call.func + if isinstance(func, ast.Attribute): + receiver = func.value.id if isinstance(func.value, ast.Name) else None + # An unbound `Path.read_text(p)` puts the instance in slot 0, and + # `pathlib.Path.read_text(p)` is the same call fully qualified. + shift = 1 if _is_path_class(receiver, modules) or _is_path_attr(func.value) else 0 + if func.attr in GUARDED_METHODS: + if func.attr == "read_text" and not shift and call.args: + first = call.args[0] + # Bound read_text takes encoding first, so None or "locale" + # there is a platform-default read. Any other positional means + # the receiver is importlib.metadata's Distribution, whose + # argument is a filename and which takes no encoding at all. + if isinstance(first, ast.Constant) and first.value in PLATFORM_DEFAULT_ENCODINGS: + return "read_text()" + return None + position = ENCODING_POSITION[func.attr] + shift + return None if _pins_encoding(call, position) else f"{func.attr}()" + if func.attr == "open": + # io.open and builtins.open ARE the builtin, so they take the + # builtin's argument positions and the same platform default. + if receiver is not None and _origin_root(receiver, modules) in BUILTIN_OPEN_MODULES: + if not _is_text(call, 1) or _pins_encoding(call, ENCODING_POSITION["open"]): + return None + return f"{receiver}.open()" + compressed = _compressed_key(receiver, modules) if receiver else None + if compressed is not None: + mode = _open_mode(call, 1) + if mode is UNKNOWN_MODE or "t" not in str(mode): + return None # "rb" default, so binary unless asked otherwise + return ( + None + if _pins_encoding(call, COMPRESSED_OPENERS[compressed]) + else f"{compressed}.open()" + ) + # Any other module receiver is somebody else's opener: tarfile.open + # takes a compression mode, Image.open takes a binary file. Neither + # has an encoding to name, so demanding one leaves no correct edit. + if ( + receiver is not None + and receiver in modules + and not _is_path_class(receiver, modules) + ): + return None + if not _is_text(call, shift): + return None + return ( + None + if _pins_encoding(call, ENCODING_POSITION["Path.open"] + shift) + else "Path.open()" + ) + return None + if isinstance(func, ast.Name): + alias = _open_alias(func.id, modules) + # Binary handles have no encoding to name. + if alias == "builtin" and _is_text(call, 1): + return None if _pins_encoding(call, ENCODING_POSITION["open"]) else "open()" + if alias is not None and alias != "builtin": + mode = _open_mode(call, 1) + if mode is UNKNOWN_MODE or "t" not in str(mode): + return None # "rb" default, so binary unless asked otherwise + position = COMPRESSED_OPENERS[alias] + return None if _pins_encoding(call, position) else f"{alias}.open()" + return None + + +def _scan(tree: ast.Module, rel: str): + """Offenders from both rules, reported once each and in source order.""" + modules = _imported_names(tree) + visible_at = _imports_at_each_call(tree) + calls = {id(c): c for c in _import_time_calls(tree)} + calls.update({id(c): c for c in _checked_in_path_calls(tree, modules, visible_at)}) + not_paths = _non_path_names(tree) + temp_roots = _temp_rooted_names(tree) + for call in sorted(calls.values(), key = lambda c: (c.lineno, c.col_offset)): + func = call.func + if ( + isinstance(func, ast.Attribute) + and (func.attr in GUARDED_METHODS or func.attr == "open") + and isinstance(func.value, ast.Name) + and func.value.id in not_paths + ): + continue # ZipFile.open and friends have no encoding to name + expr = _path_expr(call, visible_at.get(id(call), modules)) + root = _path_root(expr) if expr is not None else None + if isinstance(root, ast.Name) and root.id in temp_roots: + continue # the run made this file, so the platform default is safe + name = _offender(call, visible_at.get(id(call), modules)) + if name is not None: + yield f"{rel}:{call.lineno}: {name}" + + +def test_checked_in_file_reads_name_an_encoding(): + offenders = [] + for path in sorted(SOURCES): + tree = ast.parse(path.read_text(encoding = "utf-8"), filename = str(path)) + offenders.extend(_scan(tree, path.relative_to(REPO).as_posix())) + assert offenders == [], ( + f"{len(offenders)} file reads in the test trees touch a checked-in file " + "with the platform default encoding, so they break on Windows as soon " + 'as that file gains a non-ASCII byte. Pass encoding = "utf-8": ' + f"{offenders[:10]}" + ) diff --git a/tests/test_studio_install_workspace_guard.py b/tests/test_studio_install_workspace_guard.py index 18678a9b4a..fa6c8afea4 100644 --- a/tests/test_studio_install_workspace_guard.py +++ b/tests/test_studio_install_workspace_guard.py @@ -15,16 +15,13 @@ SETUP_SH = REPO_ROOT / "studio" / "setup.sh" # Stubs for helpers the extracted guard block calls; mv-based replacement reproduces the venv-gone # effect without the full rollback machinery. _INSTALL_GUARD_STUBS = ( - "substep() { :; }\n" - "_start_studio_venv_replacement() {\n" - ' mv -- "$1" "$1.replaced"\n' - "}\n" + 'substep() { :; }\n_start_studio_venv_replacement() {\n mv -- "$1" "$1.replaced"\n}\n' ) def _extract_install_sh_guard_block() -> str: """Extract install.sh's venv guard block (up to the first elif) as a self-contained snippet.""" - src = INSTALL_SH.read_text() + src = INSTALL_SH.read_text(encoding = "utf-8") m = re.search( r'(if \[ -x "\$VENV_DIR/bin/python" \]; then\n.*?)elif \[ "\$_STUDIO_HOME_REDIRECT" != "env"', src, @@ -119,7 +116,7 @@ def test_default_mode_skips_sentinel_check(tmp_path): def test_install_ps1_has_matching_env_mode_guard(): - src = INSTALL_PS1.read_text() + src = INSTALL_PS1.read_text(encoding = "utf-8") block_start = src.index("if (Test-Path -LiteralPath $VenvPython)") block = src[block_start : block_start + 2000] assert ( @@ -131,7 +128,7 @@ def test_install_ps1_has_matching_env_mode_guard(): def test_setup_ps1_has_writability_probe(): - src = SETUP_PS1.read_text() + src = SETUP_PS1.read_text(encoding = "utf-8") idx = src.index("if (Test-Path -LiteralPath $_studioOverride -PathType Container)") block = src[idx : idx + 2000] assert ( @@ -193,7 +190,7 @@ def test_env_mode_passes_when_bin_unsloth_is_a_symlink(tmp_path): def test_install_ps1_sentinel_uses_pathtype_leaf(): """Remove-Item $VenvDir gate must use -PathType Leaf so a sentinel-path directory cannot satisfy it.""" - src = INSTALL_PS1.read_text() + src = INSTALL_PS1.read_text(encoding = "utf-8") block_start = src.index("if (Test-Path -LiteralPath $VenvPython)") block = src[block_start : block_start + 2000] assert ( @@ -206,7 +203,7 @@ def test_install_ps1_sentinel_uses_pathtype_leaf(): def test_setup_ps1_stale_venv_has_env_mode_guard(): """setup.ps1 stale-venv branch must gate Remove-Item $VenvDir on a custom-root Unsloth sentinel.""" - src = SETUP_PS1.read_text() + src = SETUP_PS1.read_text(encoding = "utf-8") idx = src.index("Stale venv detected") block = src[idx : idx + 1500] assert ( @@ -226,7 +223,7 @@ def test_setup_ps1_stale_venv_has_env_mode_guard(): def test_setup_sh_prebuilt_llama_cpp_has_ownership_guard(): """setup.sh prebuilt llama.cpp path must _assert_studio_owned_or_absent before install_llama_prebuilt.py.""" - src = SETUP_SH.read_text() + src = SETUP_SH.read_text(encoding = "utf-8") idx = src.index("installing prebuilt llama.cpp...") block = src[idx : idx + 2000] assert ( @@ -240,7 +237,7 @@ def test_setup_sh_prebuilt_llama_cpp_has_ownership_guard(): def test_setup_ps1_prebuilt_llama_cpp_has_ownership_guard(): """setup.ps1 prebuilt llama.cpp path must Assert-StudioOwnedOrAbsent before install_llama_prebuilt.py.""" - src = SETUP_PS1.read_text() + src = SETUP_PS1.read_text(encoding = "utf-8") idx = src.index("installing prebuilt llama.cpp bundle (preferred path)") block = src[idx : idx + 2000] assert ( @@ -266,9 +263,9 @@ def test_env_mode_passes_when_venv_marker_present(tmp_path): """install.sh env-mode guard must accept the in-VENV .unsloth-studio-owned marker as a sentinel.""" studio_home = tmp_path / "ws" res = _run_install_guard(studio_home, redirect = "env", create_venv_marker = True) - assert res.returncode == 0, ( - f"in-VENV marker must allow cleanup; " f"stdout={res.stdout!r} stderr={res.stderr!r}" - ) + assert ( + res.returncode == 0 + ), f"in-VENV marker must allow cleanup; stdout={res.stdout!r} stderr={res.stderr!r}" assert "RESULT=ok" in res.stdout assert not (studio_home / "unsloth_studio").exists() @@ -318,16 +315,15 @@ def test_env_mode_blocks_when_bin_unsloth_is_broken_symlink(tmp_path): text = True, capture_output = True, ) - assert res.returncode != 0, ( - "broken symlink at bin/unsloth must NOT pass; " - f"stdout={res.stdout!r} stderr={res.stderr!r}" - ) + assert ( + res.returncode != 0 + ), f"broken symlink at bin/unsloth must NOT pass; stdout={res.stdout!r} stderr={res.stderr!r}" assert (venv / "important.txt").is_file() def test_install_sh_writes_venv_marker_after_uv_venv(): """install.sh must write .unsloth-studio-owned into $VENV_DIR right after `uv venv` succeeds.""" - src = INSTALL_SH.read_text() + src = INSTALL_SH.read_text(encoding = "utf-8") create_idx = src.index('run_install_cmd "create venv" uv venv "$VENV_DIR"') tail = src[create_idx : create_idx + 600] assert ( @@ -337,7 +333,7 @@ def test_install_sh_writes_venv_marker_after_uv_venv(): def test_install_ps1_writes_venv_marker_after_uv_venv(): """install.ps1 must write .unsloth-studio-owned into $VenvDir after `uv venv` succeeds.""" - src = INSTALL_PS1.read_text() + src = INSTALL_PS1.read_text(encoding = "utf-8") venv_create = src.index("uv venv $VenvDir --python") tail = src[venv_create : venv_create + 1500] assert ( @@ -347,7 +343,7 @@ def test_install_ps1_writes_venv_marker_after_uv_venv(): def test_install_ps1_guard_accepts_venv_marker(): """install.ps1 env-mode guard must accept the in-VENV .unsloth-studio-owned marker as a sentinel.""" - src = INSTALL_PS1.read_text() + src = INSTALL_PS1.read_text(encoding = "utf-8") block_start = src.index("if (Test-Path -LiteralPath $VenvPython)") block = src[block_start : block_start + 2000] assert ( @@ -357,7 +353,7 @@ def test_install_ps1_guard_accepts_venv_marker(): def test_setup_helpers_gate_on_canonical_custom_root(): """setup.sh/setup.ps1 ownership guards must gate on a canonical custom-vs-legacy root comparison.""" - sh_src = SETUP_SH.read_text() + sh_src = SETUP_SH.read_text(encoding = "utf-8") sh_idx = sh_src.index("_assert_studio_owned_or_absent() {") sh_func = sh_src[sh_idx : sh_idx + 600] assert ( @@ -369,7 +365,7 @@ def test_setup_helpers_gate_on_canonical_custom_root(): and "_STUDIO_HOME_IS_CUSTOM=" in sh_src ), "setup.sh must compute the canonical custom-root flag" - ps_src = SETUP_PS1.read_text() + ps_src = SETUP_PS1.read_text(encoding = "utf-8") ps_idx = ps_src.index("function Assert-StudioOwnedOrAbsent") ps_func = ps_src[ps_idx : ps_idx + 800] assert ( @@ -382,7 +378,7 @@ def test_setup_helpers_gate_on_canonical_custom_root(): def test_setup_ps1_inplace_git_sync_marks_studio_owned(): """setup.ps1 in-place git-sync branch must Mark-StudioOwned after a successful sync.""" - src = SETUP_PS1.read_text() + src = SETUP_PS1.read_text(encoding = "utf-8") inplace_idx = src.index('Test-Path -LiteralPath (Join-Path $LlamaCppDir ".git")') # The in-place branch ends just before the temp-dir clone branch. clone_idx = src.index("Cloning llama.cpp @", inplace_idx) @@ -397,7 +393,7 @@ def test_setup_ps1_inplace_git_sync_marks_studio_owned(): def test_setup_ps1_inplace_git_sync_asserts_studio_owned_before_mutation(): """setup.ps1 in-place git-sync must Assert-StudioOwnedOrAbsent before any destructive git op.""" - src = SETUP_PS1.read_text() + src = SETUP_PS1.read_text(encoding = "utf-8") inplace_idx = src.index('Test-Path -LiteralPath (Join-Path $LlamaCppDir ".git")') clone_idx = src.index("Cloning llama.cpp @", inplace_idx) inplace_block = src[inplace_idx:clone_idx] @@ -410,7 +406,7 @@ def test_setup_ps1_inplace_git_sync_asserts_studio_owned_before_mutation(): def _extract_check_health_function() -> str: - src = INSTALL_SH.read_text() + src = INSTALL_SH.read_text(encoding = "utf-8") fn_start = src.index("_check_health() {") fn_end = src.index("\n}\n", fn_start) + 2 return src[fn_start:fn_end] @@ -498,7 +494,7 @@ def test_check_health_handles_arbitrary_id_token(): def test_install_ps1_test_studio_health_verifies_studio_root_id(): """install.ps1 Test-StudioHealth must compare studio_root_id against baked $_ExpectedStudioRootId.""" - src = INSTALL_PS1.read_text() + src = INSTALL_PS1.read_text(encoding = "utf-8") fn_start = src.index("function Test-StudioHealth") fn_end = src.index("\n}\n", fn_start) + 2 fn = src[fn_start:fn_end] @@ -510,7 +506,7 @@ def test_install_ps1_test_studio_health_verifies_studio_root_id(): def test_install_ps1_bakes_studio_root_id_into_launcher(): """install.ps1 must persist a CSPRNG id at share/studio_install_id and bake it as $_ExpectedStudioRootId.""" - src = INSTALL_PS1.read_text() + src = INSTALL_PS1.read_text(encoding = "utf-8") assert "$_studioRootId" in src, "install.ps1 must compute $_studioRootId for the launcher" assert ( '"share"' in src and "studio_install_id" in src @@ -526,7 +522,7 @@ def test_install_ps1_bakes_studio_root_id_into_launcher(): def test_health_endpoint_exposes_studio_root_id_not_raw_path(): """/api/health must expose studio_root_id (hex digest), NOT the raw path (info disclosure on -H 0.0.0.0).""" main_py = REPO_ROOT / "studio" / "backend" / "main.py" - src = main_py.read_text() + src = main_py.read_text(encoding = "utf-8") health_idx = src.index('@app.get("/api/health")') # Slice up to the next top-level @app. so a growing body stays in scope. next_app_idx = src.find("\n@app.", health_idx + 1) @@ -542,7 +538,7 @@ def test_health_endpoint_exposes_studio_root_id_not_raw_path(): def test_install_sh_bakes_studio_root_id_into_launcher(): """install.sh must persist the id at share/studio_install_id and bake it into the launcher for ALL modes.""" - src = INSTALL_SH.read_text() + src = INSTALL_SH.read_text(encoding = "utf-8") assert ( "_css_studio_root_id" in src ), "install.sh must compute _css_studio_root_id for the launcher" @@ -568,8 +564,10 @@ def test_tauri_preflight_scrubs_studio_home_env(): preflight_root / "preflight.rs", *(preflight_root / "preflight").glob("*.rs"), ] - preflight = "\n".join(p.read_text() for p in preflight_paths if p.exists()) - commands = (REPO_ROOT / "studio" / "src-tauri" / "src" / "commands.rs").read_text() + preflight = "\n".join(p.read_text(encoding = "utf-8") for p in preflight_paths if p.exists()) + commands = (REPO_ROOT / "studio" / "src-tauri" / "src" / "commands.rs").read_text( + encoding = "utf-8" + ) # Expect 2 scrubs in preflight (run_cli_probe + probe_cli_capability), 1 in commands. assert ( preflight.count('cmd.env_remove("UNSLOTH_STUDIO_HOME")') >= 2 @@ -587,7 +585,7 @@ def test_tauri_preflight_scrubs_studio_home_env(): def test_install_sh_shim_uses_atomic_replace(): """install.sh shim install must use ln -sfn for atomic replace (rm+ln left a missing-shim window).""" - src = INSTALL_SH.read_text() + src = INSTALL_SH.read_text(encoding = "utf-8") shim_idx = src.index('_shim_path="$_LOCAL_BIN/unsloth"') block = src[shim_idx : shim_idx + 1500] assert ( @@ -600,7 +598,7 @@ def test_install_sh_shim_uses_atomic_replace(): def test_install_sh_create_shortcuts_seeds_id_from_csprng_with_python_fallback(tmp_path): """_create_shortcuts seeds ids from /dev/urandom (python3 secrets fallback) and is re-run idempotent.""" - src = INSTALL_SH.read_text() + src = INSTALL_SH.read_text(encoding = "utf-8") fn_start = src.index('_css_data_dir="$DATA_DIR"') block = src[fn_start : fn_start + 3000] urandom_idx = block.index("od -An -N32 -tx1 /dev/urandom") @@ -645,7 +643,7 @@ def test_install_sh_create_shortcuts_seeds_id_from_csprng_with_python_fallback(t def test_install_sh_create_shortcuts_fails_fast_when_no_entropy(): """With no entropy source, _create_shortcuts must `return 1` not bake an empty studio_root_id.""" - src = INSTALL_SH.read_text() + src = INSTALL_SH.read_text(encoding = "utf-8") fn_start = src.index('_css_data_dir="$DATA_DIR"') block = src[fn_start : fn_start + 3000] assert ( @@ -661,7 +659,7 @@ def test_install_sh_create_shortcuts_fails_fast_when_no_entropy(): def test_install_sh_bakes_installed_is_env_mode_flag_in_launcher(): """install.sh must bake the install-time mode into the launcher so a sourced studio.conf can't flip it.""" - src = INSTALL_SH.read_text() + src = INSTALL_SH.read_text(encoding = "utf-8") assert ( "_INSTALLED_IS_ENV_MODE='@@INSTALLED_IS_ENV_MODE@@'" in src ), "launcher heredoc must declare _INSTALLED_IS_ENV_MODE='@@INSTALLED_IS_ENV_MODE@@'" @@ -676,7 +674,7 @@ def test_install_sh_bakes_installed_is_env_mode_flag_in_launcher(): def test_install_sh_launcher_gates_port_file_on_baked_flag_not_runtime_env(): """Launcher PORT_FILE/LOCK_DIR must gate on baked $_INSTALLED_IS_ENV_MODE, not runtime $UNSLOTH_STUDIO_HOME.""" - src = INSTALL_SH.read_text() + src = INSTALL_SH.read_text(encoding = "utf-8") heredoc_start = src.index("cat > \"$_css_launcher\" << 'LAUNCHER_EOF'") heredoc_end = src.index("LAUNCHER_EOF\n", heredoc_start) heredoc = src[heredoc_start:heredoc_end] @@ -724,7 +722,7 @@ def test_install_sh_launcher_gates_port_file_on_baked_flag_not_runtime_env(): def test_main_py_studio_root_id_caches_at_module_load(): """_studio_root_id() must read the id once at module load and reuse it (no per-poll FS/hash work).""" - main_py = (REPO_ROOT / "studio" / "backend" / "main.py").read_text() + main_py = (REPO_ROOT / "studio" / "backend" / "main.py").read_text(encoding = "utf-8") assert ( "_STUDIO_ROOT_ID_CACHE: str = _read_studio_install_id()" in main_py ), "main.py must populate _STUDIO_ROOT_ID_CACHE from _read_studio_install_id() at module load" @@ -785,7 +783,7 @@ def test_llama_cpp_search_roots_handles_studio_root_oserror(): holds the handler so the two never disagree on which root is legacy.""" llama_cpp = ( REPO_ROOT / "studio" / "backend" / "core" / "inference" / "llama_cpp.py" - ).read_text() + ).read_text(encoding = "utf-8") def _method_body(name: str) -> str: # Whole method body (def to next sibling def) so the check survives growth. @@ -830,7 +828,7 @@ def test_install_sh_install_id_survives_symlinked_studio_home(tmp_path): def test_install_sh_substitutes_root_id_before_data_dir(): """sed must bake the non-user-controlled placeholders before @@DATA_DIR@@ so a crafted $DATA_DIR isn't mutated.""" - src = INSTALL_SH.read_text() + src = INSTALL_SH.read_text(encoding = "utf-8") root_id_idx = src.index("s|@@STUDIO_ROOT_ID@@|$_css_studio_root_id|g") env_mode_idx = src.index("s|@@INSTALLED_IS_ENV_MODE@@|$_css_is_env_mode|g") data_dir_idx = src.index("s|@@DATA_DIR@@|$_sed_safe|g") @@ -845,13 +843,15 @@ def test_install_sh_substitutes_root_id_before_data_dir(): def test_install_sh_root_id_pass_does_not_mutate_user_data_dir(tmp_path): """A $DATA_DIR containing the literal @@STUDIO_ROOT_ID@@ must survive the placeholder-first sed passes.""" - src = INSTALL_SH.read_text() + src = INSTALL_SH.read_text(encoding = "utf-8") heredoc_start = src.index("cat > \"$_css_launcher\" << 'LAUNCHER_EOF'") heredoc_body_start = src.index("\n", heredoc_start) + 1 heredoc_body_end = src.index("LAUNCHER_EOF\n", heredoc_start) template = src[heredoc_body_start:heredoc_body_end] launcher_path = tmp_path / "launch.sh" - launcher_path.write_text(template) + # template comes out of install.sh, so it carries whatever non-ASCII that + # file holds and cp1252 cannot encode it back out. + launcher_path.write_text(template, encoding = "utf-8") # sed order: root-id first, then data-dir. weird_data_dir = "/tmp/with-@@STUDIO_ROOT_ID@@/share" root_id = "deadbeef" * 8 @@ -866,7 +866,8 @@ sed "s|@@DATA_DIR@@|$_sed_safe|g" "{launcher_path}" > "{launcher_path}.tmp" \\ && mv "{launcher_path}.tmp" "{launcher_path}" """ subprocess.run(["bash", "-c", script], check = True) - final = launcher_path.read_text() + # written as utf-8 just above, and the template carries U+2500. + final = launcher_path.read_text(encoding = "utf-8") assert ( f"DATA_DIR='{weird_data_dir}'" in final ), f"DATA_DIR must be preserved verbatim (no @@STUDIO_ROOT_ID@@ mutation); got: {final[:500]}" @@ -877,7 +878,7 @@ sed "s|@@DATA_DIR@@|$_sed_safe|g" "{launcher_path}" > "{launcher_path}.tmp" \\ def test_install_ps1_install_id_file_layout_matches_backend_read_path(): """install.ps1 must write the id at share/studio_install_id where the backend reads it, idempotently.""" - src = INSTALL_PS1.read_text() + src = INSTALL_PS1.read_text(encoding = "utf-8") id_idx = src.index('$_studioIdDir = Join-Path $StudioHome "share"') context = src[id_idx : id_idx + 1500] assert ( diff --git a/tests/test_studio_root_resilience.py b/tests/test_studio_root_resilience.py index 779ff2f3f1..66195d11fe 100644 --- a/tests/test_studio_root_resilience.py +++ b/tests/test_studio_root_resilience.py @@ -64,7 +64,7 @@ def test_kill_orphan_catches_oserror_from_studio_root(): """Cleanup must not crash when studio_root() raises. _kill_orphaned_servers resolves the install root through the shared _resolved_studio_root_and_is_legacy() classifier, which swallows (ImportError, OSError, ValueError) on the probe.""" - src = LLAMA_CPP.read_text() + src = LLAMA_CPP.read_text(encoding = "utf-8") # Cleanup delegates to the shared classifier rather than importing studio_root inline. assert "LlamaCppBackend._resolved_studio_root_and_is_legacy()" in _method_body( src, "_kill_orphaned_servers" @@ -85,7 +85,7 @@ def _exec_search_roots_block( """Run _find_llama_server_binary's search_roots derivation -- plus the shared _resolved_studio_root_and_is_legacy() classifier it delegates to -- with a controlled studio_root() and resolve(), without importing the heavy module.""" - src = LLAMA_CPP.read_text() + src = LLAMA_CPP.read_text(encoding = "utf-8") # Shared root classifier (holds the defensive try/except for studio_root()). # End the slice at the next sibling def/decorator at the same indent rather # than the literal "@staticmethod" string, so a future docstring mentioning a diff --git a/tests/test_tool_mask_zoo_compat.py b/tests/test_tool_mask_zoo_compat.py index 6212b6f807..84c032da3b 100644 --- a/tests/test_tool_mask_zoo_compat.py +++ b/tests/test_tool_mask_zoo_compat.py @@ -15,7 +15,7 @@ RL_REPLACEMENTS_SOURCE_PATH = os.path.join(REPO_ROOT, "unsloth", "models", "rl_r def _read(path: str) -> str: - with open(path, "r") as fh: + with open(path, "r", encoding = "utf-8") as fh: return fh.read() diff --git a/tests/utils/test_prepare_inputs_leftpad.py b/tests/utils/test_prepare_inputs_leftpad.py index 2bfd763279..9a64103770 100644 --- a/tests/utils/test_prepare_inputs_leftpad.py +++ b/tests/utils/test_prepare_inputs_leftpad.py @@ -46,7 +46,7 @@ WIRED_MODEL_FILES = [ def _load_function(): - tree = ast.parse(LLAMA_PY.read_text()) + tree = ast.parse(LLAMA_PY.read_text(encoding = "utf-8")) for node in ast.walk(tree): if isinstance(node, ast.FunctionDef) and node.name == FUNC_NAME: return node @@ -207,7 +207,7 @@ def test_model_families_stay_wired_to_shared_prepare_inputs(): path = REPO_ROOT / "unsloth" / "models" / fname if not path.exists(): continue - if "fix_prepare_inputs_for_generation(" not in path.read_text(): + if "fix_prepare_inputs_for_generation(" not in path.read_text(encoding = "utf-8"): missing.append(fname) assert not missing, ( "these model files no longer call fix_prepare_inputs_for_generation, " diff --git a/tests/utils/test_rope_scaling_drift.py b/tests/utils/test_rope_scaling_drift.py index b2ec1e5a20..eba89734f7 100644 --- a/tests/utils/test_rope_scaling_drift.py +++ b/tests/utils/test_rope_scaling_drift.py @@ -52,7 +52,7 @@ MAX_POS = 131072 def _load_class_init(): - tree = ast.parse(LLAMA_PY.read_text()) + tree = ast.parse(LLAMA_PY.read_text(encoding = "utf-8")) for node in ast.walk(tree): if isinstance(node, ast.ClassDef) and node.name == CLASS_NAME: for sub in node.body: @@ -96,7 +96,7 @@ def _iter_names_and_calls(node): def _find_method(source_path, class_name, method_name): - for node in ast.walk(ast.parse(source_path.read_text())): + for node in ast.walk(ast.parse(source_path.read_text(encoding = "utf-8"))): if isinstance(node, ast.ClassDef) and node.name == class_name: for sub in node.body: if isinstance(sub, ast.FunctionDef) and sub.name == method_name: @@ -105,7 +105,7 @@ def _find_method(source_path, class_name, method_name): def _find_function(source_path, function_name): - for node in ast.walk(ast.parse(source_path.read_text())): + for node in ast.walk(ast.parse(source_path.read_text(encoding = "utf-8"))): if isinstance(node, ast.FunctionDef) and node.name == function_name: return node return None diff --git a/unsloth_cli/__init__.py b/unsloth_cli/__init__.py index 121b26f03f..703a6f1f60 100644 --- a/unsloth_cli/__init__.py +++ b/unsloth_cli/__init__.py @@ -4,6 +4,28 @@ import os as _os import sys as _sys +# Are we the `unsloth` console script, rather than a library import? Both the +# stream guard below and the `-np` rewrite further down are entry-point +# behaviour and must not reach into a host application that imports us. +_entry_base = _os.path.basename(_sys.argv[0]).lower() if _sys.argv else "" +_is_entry_point = _entry_base in {"unsloth", "unsloth.exe"} + +# Typer renders help via rich, whose box characters cp1252 and cp437 cannot encode, +# so `unsloth --help` dies once stdout is a pipe or a file. Windows gets UTF-8, as +# unsloth/__init__ already does; elsewhere the caller's encoding is kept and only +# the error handler is relaxed, so an explicit PYTHONIOENCODING still picks the +# bytes and only loses unencodable glyphs. Before typer, which binds the stream. +if _is_entry_point: + _to_utf8 = _sys.platform == "win32" + for _name in ("stdout", "stderr"): + _stream = getattr(_sys, _name, None) + try: + if "utf" not in (_stream.encoding or "").lower(): + _stream.reconfigure(encoding = "utf-8" if _to_utf8 else None, errors = "replace") + except Exception: + pass + del _name, _stream, _to_utf8 + import typer from importlib.metadata import version as package_version, PackageNotFoundError @@ -22,10 +44,9 @@ from unsloth_cli.commands.studio import ( # Canonicalise `-np` only under the `unsloth` console-script; # third-party scripts that import unsloth_cli keep their argv intact. -_entry_base = _os.path.basename(_sys.argv[0]).lower() if _sys.argv else "" -if _entry_base in {"unsloth", "unsloth.exe"}: +if _is_entry_point: _expand_attached_np_short() -del _entry_base +del _entry_base, _is_entry_point def show_version(value: bool): diff --git a/unsloth_cli/tests/test_start.py b/unsloth_cli/tests/test_start.py index 4cdb37e7fa..dd0edfaa15 100644 --- a/unsloth_cli/tests/test_start.py +++ b/unsloth_cli/tests/test_start.py @@ -587,7 +587,9 @@ def test_write_codex_config_profile(tmp_path, monkeypatch): assert catalog["models"][0]["supports_reasoning_summary_parameter"] is False assert catalog["models"][0]["supports_parallel_tool_calls"] is False - assert catalog["models"][0]["base_instructions"] == start._CODEX_FALLBACK_PROMPT.read_text() + assert catalog["models"][0]["base_instructions"] == start._CODEX_FALLBACK_PROMPT.read_text( + encoding = "utf-8" + ) config = _parse_toml((tmp_path / "config.toml").read_text()) assert config["model_providers"]["unsloth_api"]["env_key"] == "UNSLOTH_STUDIO_AUTH_TOKEN" @@ -631,7 +633,7 @@ def test_write_codex_subagent_bridge_keeps_parent_credentials_out(tmp_path, monk tmp_path, yolo = False, ) - assert json.loads(path.read_text()) == { + assert json.loads(path.read_text(encoding = "utf-8")) == { "api_key": "private-token", "codex_home": str(tmp_path / "child"), "bypass_permissions": False,