diff --git a/scripts/lint_workflow_triggers.py b/scripts/lint_workflow_triggers.py index 0688f6c65c..8f22fcaf45 100644 --- a/scripts/lint_workflow_triggers.py +++ b/scripts/lint_workflow_triggers.py @@ -52,14 +52,14 @@ def _normalise_on(on_field): def _load_workflow(path: Path): try: - return yaml.safe_load(path.read_text()) + return yaml.safe_load(path.read_text(encoding = "utf-8")) except Exception as exc: print(f"ERROR: failed to parse {path}: {exc}", file = sys.stderr) sys.exit(2) def _extract_cache_keys(path: Path) -> list[str]: - text = path.read_text() + text = path.read_text(encoding = "utf-8") keys: list[str] = [] for m in re.finditer(r"(?:^|\n)\s*key:\s*([^\n]+)", text): keys.append(m.group(1).strip()) @@ -104,7 +104,7 @@ def main() -> int: for t in RESTRICTED_TRIGGERS: if t in triggers: - text = path.read_text() + text = path.read_text(encoding = "utf-8") if "lint:workflow_triggers-allow-workflow_run" not in text: findings.append( f"{path.name}: RESTRICTED trigger '{t}' requires an " diff --git a/studio/backend/tests/test_cloudflare_tunnel.py b/studio/backend/tests/test_cloudflare_tunnel.py index 2094d15066..8d19f09bae 100644 --- a/studio/backend/tests/test_cloudflare_tunnel.py +++ b/studio/backend/tests/test_cloudflare_tunnel.py @@ -915,17 +915,17 @@ def _argparse_default(source, option): def test_run_server_cloudflare_default_off(): - defaults = _func_param_defaults(_RUN_PY.read_text(), "run_server") + defaults = _func_param_defaults(_RUN_PY.read_text(encoding = "utf-8"), "run_server") assert "cloudflare" in defaults assert defaults["cloudflare"] is None def test_argparse_cloudflare_default_off(): - assert _argparse_default(_RUN_PY.read_text(), "--cloudflare") is None + assert _argparse_default(_RUN_PY.read_text(encoding = "utf-8"), "--cloudflare") is None def test_verify_global_reachability_marks_private_address_unreachable(): - src = _RUN_PY.read_text() + src = _RUN_PY.read_text(encoding = "utf-8") tree = ast.parse(src) func_src = next( ast.get_source_segment(src, n) @@ -949,7 +949,7 @@ def test_verify_global_reachability_marks_private_address_unreachable(): def test_run_server_registers_tunnel_atexit_backstop(): # An abnormal exit (exception after startup -> sys.exit) bypasses # _graceful_shutdown; an atexit backstop must still stop the tunnel. - src = _RUN_PY.read_text() + src = _RUN_PY.read_text(encoding = "utf-8") assert "atexit.register(stop_studio_tunnel)" in src @@ -965,7 +965,7 @@ def _run_print_cloudflare_line( color = False, ): """Exec _print_cloudflare_line without importing run.py's heavy deps.""" - src = _RUN_PY.read_text() + src = _RUN_PY.read_text(encoding = "utf-8") tree = ast.parse(src) func_src = next( ast.get_source_segment(src, n) diff --git a/studio/backend/tests/test_consent_gate.py b/studio/backend/tests/test_consent_gate.py index 181e0c9fad..c87662edc1 100644 --- a/studio/backend/tests/test_consent_gate.py +++ b/studio/backend/tests/test_consent_gate.py @@ -402,7 +402,7 @@ class TestWorkersWireTheGate: ], ) def test_worker_invokes_gate(self, rel): - src = (Path(__file__).resolve().parent.parent / rel).read_text() + src = (Path(__file__).resolve().parent.parent / rel).read_text(encoding = "utf-8") assert "evaluate_remote_code_consent" in src assert "remote_code_blocked" in src assert ".blocked" in src @@ -410,14 +410,14 @@ class TestWorkersWireTheGate: def test_mlx_training_path_gates_before_load(self): # The Apple-Silicon path returns before run_training_process's gate, so it must # scan before FastMLXModel.from_pretrained runs repo code. - src = (_BACKEND / "core/training/worker.py").read_text() + src = (_BACKEND / "core/training/worker.py").read_text(encoding = "utf-8") head = src[: src.index("FastMLXModel.from_pretrained(")] assert "evaluate_remote_code_consent" in head def test_lora_base_model_is_gated(self): # Inference + export expand the consent scan to the LoRA base model's code. for rel in ("core/inference/worker.py", "core/export/worker.py"): - src = (_BACKEND / rel).read_text() + src = (_BACKEND / rel).read_text(encoding = "utf-8") assert "evaluate_remote_code_consent" in src assert "get_base_model_from_lora" in src or "mc.base_model" in src @@ -431,12 +431,12 @@ class TestWorkersWireTheGate: "core/training/worker.py", "core/export/worker.py", ): - src = (_BACKEND / rel).read_text() + src = (_BACKEND / rel).read_text(encoding = "utf-8") assert "get_base_model_from_lora_identifier" in src, rel def test_embedding_training_path_gates_before_load(self): # The embedding pipeline must run the malware + consent gates before loading, like the other paths. - src = (_BACKEND / "core/training/worker.py").read_text() + src = (_BACKEND / "core/training/worker.py").read_text(encoding = "utf-8") start = src.index("def _run_embedding_training(") end = src.index("FastSentenceTransformer.from_pretrained(", start) region = src[start:end] @@ -505,7 +505,9 @@ class TestStructuredFindingsForDialog: assert d.findings and d.fingerprint # structured findings for the UI def test_scan_route_uses_preflight(self): - src = (Path(__file__).resolve().parent.parent / "routes/models.py").read_text() + src = (Path(__file__).resolve().parent.parent / "routes/models.py").read_text( + encoding = "utf-8" + ) assert "remote-code-scan" in src # The scan route pins one combined fingerprint over adapter + base, so adapter code is reviewed and approvable too. assert "preflight_remote_code_consent_for_targets" in src @@ -636,7 +638,7 @@ class TestStructuredFindingsForDialog: ], ) def test_fingerprint_threaded_to_worker(self, rel): - src = (Path(__file__).resolve().parent.parent / rel).read_text() + src = (Path(__file__).resolve().parent.parent / rel).read_text(encoding = "utf-8") assert "approved_remote_code_fingerprint" in src # The per-user approval cache rides the same path as the fingerprint. assert "subject" in src @@ -738,7 +740,7 @@ class TestNemotronGateUsesTrustCheck: ], ) def test_worker_nemotron_block_calls_trust_check(self, rel): - src = (_BACKEND / rel).read_text() + src = (_BACKEND / rel).read_text(encoding = "utf-8") assert "_NEMOTRON_TRUST_SUBSTRINGS" in src assert "is_trusted_org_repo(" in src @@ -1525,6 +1527,6 @@ class TestDiscardRemoteCodeDownload: assert res == {"deleted": False, "reason": "not_cached"} def test_route_source_reports_created_by_scan(self): - src = (_BACKEND / "routes/models.py").read_text() + src = (_BACKEND / "routes/models.py").read_text(encoding = "utf-8") assert "created_by_scan" in src assert "discard-remote-code" in src diff --git a/studio/backend/tests/test_cpu_threads.py b/studio/backend/tests/test_cpu_threads.py index 9d8795b6c0..eb3c021ad5 100644 --- a/studio/backend/tests/test_cpu_threads.py +++ b/studio/backend/tests/test_cpu_threads.py @@ -120,7 +120,7 @@ def _ast_line_of_platform_compat_import(source: str) -> int: # run.py and main.py. Robust to formatting / line shifts. @pytest.mark.parametrize("entry_point", [_RUN_PY, _MAIN_PY]) def test_cpu_thread_configuration_runs_before_backend_imports(entry_point): - source = entry_point.read_text() + source = entry_point.read_text(encoding = "utf-8") call_line = _ast_line_of_configure_call(source) compat_line = _ast_line_of_platform_compat_import(source) assert call_line < compat_line, ( diff --git a/studio/backend/tests/test_data_recipe_seed.py b/studio/backend/tests/test_data_recipe_seed.py index 58bbd24061..1b6fe27bfc 100644 --- a/studio/backend/tests/test_data_recipe_seed.py +++ b/studio/backend/tests/test_data_recipe_seed.py @@ -11,7 +11,7 @@ import pytest def _seed_route_source() -> str: return ( Path(__file__).resolve().parent.parent / "routes" / "data_recipe" / "seed.py" - ).read_text() + ).read_text(encoding = "utf-8") def test_seed_inspect_load_kwargs_disables_remote_code_execution(): diff --git a/studio/backend/tests/test_desktop_auth.py b/studio/backend/tests/test_desktop_auth.py index 591d44b736..b2180d4357 100644 --- a/studio/backend/tests/test_desktop_auth.py +++ b/studio/backend/tests/test_desktop_auth.py @@ -123,7 +123,7 @@ def test_ensure_default_admin_does_not_recreate_bootstrap_for_existing_admin(): def test_ensure_default_admin_loads_existing_bootstrap_after_restart(monkeypatch): created = storage.ensure_default_admin() - bootstrap_pw = storage._BOOTSTRAP_PW_PATH.read_text().strip() + bootstrap_pw = storage._BOOTSTRAP_PW_PATH.read_text(encoding = "utf-8").strip() monkeypatch.setattr(storage, "_bootstrap_password", None) created_again = storage.ensure_default_admin() @@ -136,12 +136,12 @@ def test_ensure_default_admin_loads_existing_bootstrap_after_restart(monkeypatch def test_ensure_default_admin_does_not_generate_for_empty_existing_bootstrap(): seed_user() - storage._BOOTSTRAP_PW_PATH.write_text(" \n") + storage._BOOTSTRAP_PW_PATH.write_text(" \n", encoding = "utf-8") created = storage.ensure_default_admin() assert created is False - assert storage._BOOTSTRAP_PW_PATH.read_text() == " \n" + assert storage._BOOTSTRAP_PW_PATH.read_text(encoding = "utf-8") == " \n" assert storage.get_bootstrap_password() is None @@ -649,7 +649,7 @@ def test_desktop_auth_provision_has_bounded_timeout(): rs_path = ( Path(__file__).resolve().parents[3] / "studio" / "src-tauri" / "src" / "desktop_auth.rs" ) - src = rs_path.read_text() + src = rs_path.read_text(encoding = "utf-8") start = src.index("async fn provision_desktop_auth(") depth = 0 body_start = src.index("{", start) diff --git a/studio/backend/tests/test_gguf_load_cache_reuse.py b/studio/backend/tests/test_gguf_load_cache_reuse.py index 0ab998af39..ccbe50bcb9 100644 --- a/studio/backend/tests/test_gguf_load_cache_reuse.py +++ b/studio/backend/tests/test_gguf_load_cache_reuse.py @@ -809,7 +809,9 @@ class TestLoadHubDownloadExclusion: asyncio.run(scenario()) def test_load_marker_precedes_hub_guard_and_unload(self): - source = (Path(__file__).resolve().parent.parent / "routes" / "inference.py").read_text() + source = (Path(__file__).resolve().parent.parent / "routes" / "inference.py").read_text( + encoding = "utf-8" + ) # _load_model_impl has more than one `if config.is_gguf:`, so anchor on # the branch that actually owns the load marker rather than the first # one in the file, which belongs to an earlier check. @@ -832,7 +834,7 @@ class TestLoadHubDownloadExclusion: ) llama_source = ( Path(__file__).resolve().parent.parent / "core" / "inference" / "llama_cpp.py" - ).read_text() + ).read_text(encoding = "utf-8") assert "@_with_gguf_load_marker\n def load_model(" in llama_source def _capture_hub_guard_require_mmproj( diff --git a/studio/backend/tests/test_host_defaults.py b/studio/backend/tests/test_host_defaults.py index 5c7129bc65..b5caba7573 100644 --- a/studio/backend/tests/test_host_defaults.py +++ b/studio/backend/tests/test_host_defaults.py @@ -64,7 +64,7 @@ def test_run_server_default_host_is_loopback(): 0.0.0.0 exposes the service on all interfaces; loopback is the least-permissive default. Users needing network access pass -H 0.0.0.0. """ - source = _RUN_PY.read_text() + source = _RUN_PY.read_text(encoding = "utf-8") defaults = _parse_function_param_defaults(source, "run_server") assert "host" in defaults, "run_server() must have a 'host' parameter with a default" host_default = defaults["host"] @@ -81,7 +81,7 @@ def test_argparse_default_host_is_loopback(): When run.py is invoked directly (python run.py), the argparse default must match the function default so direct execution is equally safe. """ - source = _RUN_PY.read_text() + source = _RUN_PY.read_text(encoding = "utf-8") host_default = _parse_argparse_add_argument_default(source, "--host") assert host_default is not None, "Could not find add_argument('--host', ...) in run.py" assert ( diff --git a/studio/backend/tests/test_mcp_servers.py b/studio/backend/tests/test_mcp_servers.py index c5c37f098f..731823c292 100644 --- a/studio/backend/tests/test_mcp_servers.py +++ b/studio/backend/tests/test_mcp_servers.py @@ -599,7 +599,9 @@ def test_tool_xml_strip_handles_hyphenated_function_names(): from core.inference.tool_call_parser import _DEEPSEEK_OPEN_RE_SRC as _DS_OPEN_SRC - src = (Path(__file__).resolve().parent.parent / "routes/inference.py").read_text() + src = (Path(__file__).resolve().parent.parent / "routes/inference.py").read_text( + encoding = "utf-8" + ) m = _re.search(r"_TOOL_XML_RE = _re\.compile\((.*?)\n\)", src, _re.DOTALL) assert m, "could not extract _TOOL_XML_RE" ns: dict = {"_re": _re, "_DS_OPEN_SRC": _DS_OPEN_SRC} diff --git a/studio/backend/tests/test_mlx_training_worker_config.py b/studio/backend/tests/test_mlx_training_worker_config.py index 14fc0933d0..5dde69648f 100644 --- a/studio/backend/tests/test_mlx_training_worker_config.py +++ b/studio/backend/tests/test_mlx_training_worker_config.py @@ -86,7 +86,9 @@ def test_mlx_studio_rejects_unknown_scheduler(): def test_mlx_studio_keeps_hf_style_tokenizer_dual_purpose(): - source = (Path(__file__).resolve().parents[1] / "core" / "training" / "worker.py").read_text() + source = (Path(__file__).resolve().parents[1] / "core" / "training" / "worker.py").read_text( + encoding = "utf-8" + ) assert "tokenizer = tokenizer" in source assert "processor = tokenizer if is_vlm else None" not in source @@ -96,7 +98,9 @@ def test_mlx_wandb_run_config_excludes_subject_and_secrets(): # The MLX W&B run config uploads the whole config minus a sensitive set. The owner's # subject (authenticated username / API-key id) must be filtered alongside the secrets, # otherwise it lands in W&B run config even though DB history already strips it. - source = (Path(__file__).resolve().parents[1] / "core" / "training" / "worker.py").read_text() + source = (Path(__file__).resolve().parents[1] / "core" / "training" / "worker.py").read_text( + encoding = "utf-8" + ) assert ( '_wandb_sensitive = {"hf_token", "wandb_token", "s3_config", "subject"}' in source diff --git a/studio/backend/tests/test_mtp_vram_budget.py b/studio/backend/tests/test_mtp_vram_budget.py index 694d60cfc6..6c8b74fc54 100644 --- a/studio/backend/tests/test_mtp_vram_budget.py +++ b/studio/backend/tests/test_mtp_vram_budget.py @@ -320,7 +320,7 @@ class TestFitContextWithMtp: def _fit_backend(self, kv_per_token = 325_000): b = _make_backend() b._can_estimate_kv = lambda: True - b._estimate_kv_cache_bytes = lambda n, _t = None, **_k: (0 if n <= 0 else n * kv_per_token) + b._estimate_kv_cache_bytes = lambda n, _t = None, **_k: 0 if n <= 0 else n * kv_per_token return b def test_overhead_fn_lowers_context(self): @@ -347,19 +347,23 @@ class TestFitContextWithMtp: 131072, avail_mib, model, - mtp_overhead_fn = lambda c: b._estimate_mtp_overhead_bytes( - c, draft_cache_type_k = "f16", draft_cache_type_v = "f16" - ) - or 0, + mtp_overhead_fn = lambda c: ( + b._estimate_mtp_overhead_bytes( + c, draft_cache_type_k = "f16", draft_cache_type_v = "f16" + ) + or 0 + ), ) q4 = b._fit_context_to_vram( 131072, avail_mib, model, - mtp_overhead_fn = lambda c: b._estimate_mtp_overhead_bytes( - c, draft_cache_type_k = "q4_0", draft_cache_type_v = "q4_0" - ) - or 0, + mtp_overhead_fn = lambda c: ( + b._estimate_mtp_overhead_bytes( + c, draft_cache_type_k = "q4_0", draft_cache_type_v = "q4_0" + ) + or 0 + ), ) assert 0 < q4 == f16 @@ -818,9 +822,9 @@ class TestExtraArgsMtpDetection: # helper, or an env-driven tensor server (or its layer downgrade) is # needlessly reloaded (#6312). Read from disk (importing routes.inference # drags in heavy deps). - routes_src = ( - Path(__file__).resolve().parent.parent / "routes" / "inference.py" - ).read_text() + routes_src = (Path(__file__).resolve().parent.parent / "routes" / "inference.py").read_text( + encoding = "utf-8" + ) start = routes_src.index("def _request_matches_loaded_settings") end = routes_src.index("\ndef ", start + 1) body = "".join(routes_src[start:end].split()) @@ -832,9 +836,9 @@ class TestExtraArgsMtpDetection: def test_route_matcher_retries_after_drafter_not_found(self): # drafter_not_found must not report "already loaded" or the reload never # retries the download (#6459). Read source: importing routes pulls deps. - routes_src = ( - Path(__file__).resolve().parent.parent / "routes" / "inference.py" - ).read_text() + routes_src = (Path(__file__).resolve().parent.parent / "routes" / "inference.py").read_text( + encoding = "utf-8" + ) start = routes_src.index("def _request_matches_loaded_settings") end = routes_src.index("\ndef ", start + 1) body = "".join(routes_src[start:end].split()) @@ -990,7 +994,7 @@ def test_qwen36_class_regression_picks_lower_ctx_with_mtp(): strictly lower one once the MTP draft reserve is accounted for.""" b = _make_backend() b._can_estimate_kv = lambda: True - b._estimate_kv_cache_bytes = lambda n, _t = None, **_k: (0 if n <= 0 else int(n * 66_000)) + b._estimate_kv_cache_bytes = lambda n, _t = None, **_k: 0 if n <= 0 else int(n * 66_000) avail_mib = 24_000 model = int(17.9 * GIB) # UD-Q4_K_XL weights no_mtp = b._fit_context_to_vram(262144, avail_mib, model) diff --git a/studio/backend/tests/test_native_context_length.py b/studio/backend/tests/test_native_context_length.py index de1ca0649e..9417b4c751 100644 --- a/studio/backend/tests/test_native_context_length.py +++ b/studio/backend/tests/test_native_context_length.py @@ -374,7 +374,7 @@ class TestRouteCompleteness: def _load_source(self): """Read routes/inference.py source once.""" routes_path = Path(__file__).resolve().parent.parent / "routes" / "inference.py" - self._source = routes_path.read_text() + self._source = routes_path.read_text(encoding = "utf-8") def _find_construction_blocks(self, class_name: str) -> list[str]: """Extract all code blocks that construct a given response class.""" diff --git a/studio/backend/tests/test_native_template_trust_remote_code.py b/studio/backend/tests/test_native_template_trust_remote_code.py index 60dc80f64c..b61a3eb111 100644 --- a/studio/backend/tests/test_native_template_trust_remote_code.py +++ b/studio/backend/tests/test_native_template_trust_remote_code.py @@ -170,7 +170,9 @@ def test_backend_model_info_persists_trust_remote_code(): """Both backends must store ``trust_remote_code`` on their per-model info dict so ``render_native_template`` can source the consent value. Guards against the read landing on a key ``load_model`` never sets (which would silently no-op the fix).""" - inf = (Path(_BACKEND_DIR) / "core" / "inference" / "inference.py").read_text() - mlx = (Path(_BACKEND_DIR) / "core" / "inference" / "mlx_inference.py").read_text() + inf = (Path(_BACKEND_DIR) / "core" / "inference" / "inference.py").read_text(encoding = "utf-8") + mlx = (Path(_BACKEND_DIR) / "core" / "inference" / "mlx_inference.py").read_text( + encoding = "utf-8" + ) assert '"trust_remote_code": trust_remote_code,' in inf assert '"trust_remote_code": trust_remote_code,' in mlx diff --git a/studio/backend/tests/test_offline_inference_parent.py b/studio/backend/tests/test_offline_inference_parent.py index bd0014ea64..3e9f09bb2f 100644 --- a/studio/backend/tests/test_offline_inference_parent.py +++ b/studio/backend/tests/test_offline_inference_parent.py @@ -205,7 +205,7 @@ class TestTrainingWorkerProbeNoGlobalTimeout: import re from pathlib import Path - src = Path(_BACKEND_DIR, "core", "training", "worker.py").read_text() + src = Path(_BACKEND_DIR, "core", "training", "worker.py").read_text(encoding = "utf-8") m = re.search( r'if\s+"HF_HUB_OFFLINE"\s+not\s+in\s+os\.environ\s*:.*?' r"print\([^)]*HF_HUB_OFFLINE=1[^)]*\)", diff --git a/studio/backend/tests/test_recommended_folders_has_model.py b/studio/backend/tests/test_recommended_folders_has_model.py index 647d5dd3db..c034824ba0 100644 --- a/studio/backend/tests/test_recommended_folders_has_model.py +++ b/studio/backend/tests/test_recommended_folders_has_model.py @@ -32,7 +32,7 @@ def _load_has_downloaded_model(): """Return the real ``_dir_has_downloaded_model`` (plus its ``_safe_is_dir`` and ``_is_weight_bin`` deps, and the ``_WEIGHT_BIN_PREFIXES`` constant the latter reads) without importing the heavy module.""" - tree = ast.parse(_models_src.read_text()) + tree = ast.parse(_models_src.read_text(encoding = "utf-8")) wanted = {"_safe_is_dir", "_dir_has_downloaded_model", "_is_weight_bin"} body = [] for node in tree.body: diff --git a/studio/backend/tests/test_recommended_folders_permission.py b/studio/backend/tests/test_recommended_folders_permission.py index b65695ad93..4f0becf08d 100644 --- a/studio/backend/tests/test_recommended_folders_permission.py +++ b/studio/backend/tests/test_recommended_folders_permission.py @@ -36,7 +36,7 @@ _models_src = _backend_root / "routes" / "models.py" def _load_safe_is_dir(): """Return the real ``_safe_is_dir`` from routes/models.py without importing the dependency-laden module.""" - tree = ast.parse(_models_src.read_text()) + tree = ast.parse(_models_src.read_text(encoding = "utf-8")) fn = next( node for node in tree.body diff --git a/studio/backend/tests/test_sandbox_tools.py b/studio/backend/tests/test_sandbox_tools.py index 853a5a84ab..1a55c6298d 100644 --- a/studio/backend/tests/test_sandbox_tools.py +++ b/studio/backend/tests/test_sandbox_tools.py @@ -558,24 +558,24 @@ class TestSandboxCpuRlimitDefault: """Pin the default so a regression below 600s without opt-in is caught.""" def test_default_cpu_s_is_600(self): - src = (_BACKEND_ROOT / "core" / "inference" / "tools.py").read_text() + src = (_BACKEND_ROOT / "core" / "inference" / "tools.py").read_text(encoding = "utf-8") assert 'UNSLOTH_STUDIO_SANDBOX_CPU_S", "600"' in src def test_clone_newnet_removed(self): - src = (_BACKEND_ROOT / "core" / "inference" / "tools.py").read_text() + src = (_BACKEND_ROOT / "core" / "inference" / "tools.py").read_text(encoding = "utf-8") assert "_libc.unshare(0x40000000)" not in src # Explanatory comment retained. assert "CLONE_NEWNET" in src def test_nofile_env_tunable(self): - src = (_BACKEND_ROOT / "core" / "inference" / "tools.py").read_text() + src = (_BACKEND_ROOT / "core" / "inference" / "tools.py").read_text(encoding = "utf-8") # Parity with the other rlimits: must come from the env, not be hardcoded. assert "UNSLOTH_STUDIO_SANDBOX_NOFILE" in src class TestMaxBodyDefault: def test_default_is_500_mb(self): - src = (_BACKEND_ROOT / "utils" / "upload_limits.py").read_text() + src = (_BACKEND_ROOT / "utils" / "upload_limits.py").read_text(encoding = "utf-8") assert "DEFAULT_UPLOAD_LIMIT_MB = 500" in src assert "UNSLOTH_STUDIO_MAX_BODY_MB" in src diff --git a/studio/backend/tests/test_security_gate_consistency.py b/studio/backend/tests/test_security_gate_consistency.py index b5f1069f12..0c0367e979 100644 --- a/studio/backend/tests/test_security_gate_consistency.py +++ b/studio/backend/tests/test_security_gate_consistency.py @@ -43,7 +43,7 @@ def test_capability_probes_thread_the_hf_token(): offenders = [] for path in _iter_caller_files(): try: - tree = ast.parse(path.read_text()) + tree = ast.parse(path.read_text(encoding = "utf-8")) except SyntaxError: continue for node in ast.walk(tree): @@ -60,7 +60,7 @@ def test_capability_probes_thread_the_hf_token(): def test_gguf_trust_remote_code_reported_inert_not_from_yaml(): """GGUF never executes auto_map, so requires_trust_remote_code is reported via the resolver or False, never the raw YAML bool() (the round-6 regression).""" - src = (_BACKEND / "routes" / "inference.py").read_text() + src = (_BACKEND / "routes" / "inference.py").read_text(encoding = "utf-8") assert "requires_trust_remote_code = bool(" not in src, ( "Report requires_trust_remote_code via _resolve_loaded_trust_remote_code " "(non-GGUF) or set it False (GGUF); never bool(inference_config.get(...))." @@ -70,7 +70,7 @@ def test_gguf_trust_remote_code_reported_inert_not_from_yaml(): def test_capability_detection_caches_are_token_aware(): """Every capability cache is keyed by (model, token_fingerprint) so an unauthenticated miss cannot poison a later authenticated lookup (the audio-cache regression).""" - src = (_BACKEND / "utils" / "models" / "model_config.py").read_text() + src = (_BACKEND / "utils" / "models" / "model_config.py").read_text(encoding = "utf-8") offenders = [] for line in src.splitlines(): stripped = line.strip() @@ -93,7 +93,7 @@ def test_malware_and_consent_gates_cover_the_lora_base(): ] offenders = [] for rel in gated_workers: - src = (_BACKEND / rel).read_text() + src = (_BACKEND / rel).read_text(encoding = "utf-8") runs_gate = "evaluate_file_security(" in src or "evaluate_remote_code_consent" in src resolves_base = "get_base_model_from_lora_identifier(" in src or "base_model" in src if runs_gate and not resolves_base: @@ -107,7 +107,7 @@ def test_rag_embedding_path_runs_the_malware_gate(): or a flagged repo loads unscanned (bypassing the normal model-load protections).""" offenders = [] for rel in ("routes/settings.py", "core/rag/embeddings.py"): - if "evaluate_file_security(" not in (_BACKEND / rel).read_text(): + if "evaluate_file_security(" not in (_BACKEND / rel).read_text(encoding = "utf-8"): offenders.append( f"{rel} loads/persists an embedding model without evaluate_file_security" ) diff --git a/studio/backend/tests/test_ssm_runtime.py b/studio/backend/tests/test_ssm_runtime.py index bb0caa2887..b95747e56c 100644 --- a/studio/backend/tests/test_ssm_runtime.py +++ b/studio/backend/tests/test_ssm_runtime.py @@ -401,13 +401,13 @@ def test_hip_uv_source_build_uses_no_cache(monkeypatch): def test_inference_worker_calls_ensure_ssm_runtime(): - src = (_BACKEND / "core" / "inference" / "worker.py").read_text() + src = (_BACKEND / "core" / "inference" / "worker.py").read_text(encoding = "utf-8") assert "from utils.ssm_runtime import ensure_ssm_runtime" in src assert "ensure_ssm_runtime(" in src def test_inference_worker_skips_ssm_on_mlx_and_checks_lora_base(): - src = (_BACKEND / "core" / "inference" / "worker.py").read_text() + src = (_BACKEND / "core" / "inference" / "worker.py").read_text(encoding = "utf-8") # MLX (Apple Silicon) must not try to build CUDA/ROCm SSM kernels. assert 'getattr(backend, "device", None) != "mlx"' in src # A LoRA load must also check its base model, not just the adapter id. @@ -417,12 +417,12 @@ def test_inference_worker_skips_ssm_on_mlx_and_checks_lora_base(): def test_inference_worker_resolves_remote_lora_base_pre_import(): # A remote LoRA's base (from the Hub adapter_config.json) must be resolved before the # transformers import so its SSM kernels are pre-installed, not too late in _handle_load. - src = (_BACKEND / "core" / "inference" / "worker.py").read_text() + src = (_BACKEND / "core" / "inference" / "worker.py").read_text(encoding = "utf-8") assert "_remote_lora_base" in src def test_inference_worker_tiers_on_base_and_gates_lora_base_only(): - src = (_BACKEND / "core" / "inference" / "worker.py").read_text() + src = (_BACKEND / "core" / "inference" / "worker.py").read_text(encoding = "utf-8") # Tier activation runs on the resolved base, not the raw adapter id (remote-LoRA fix). assert "_activate_transformers_version(_base" in src # The gate only adds a genuine LoRA base, never a full fine-tune's recorded (unloaded) base. @@ -432,7 +432,7 @@ def test_inference_worker_tiers_on_base_and_gates_lora_base_only(): def test_inference_worker_probes_base_for_ssm_kernels(): # Both the pre-import path and _handle_load must derive SSM targets from a real model id # via ssm_probe_identifier, not the raw adapter id / local checkpoint path. - src = (_BACKEND / "core" / "inference" / "worker.py").read_text() + src = (_BACKEND / "core" / "inference" / "worker.py").read_text(encoding = "utf-8") assert src.count("ssm_probe_identifier(") >= 2 @@ -484,7 +484,7 @@ def test_pre_import_gate_is_transformers_free(): def test_pre_import_gate_skips_subdir_computation(): # The worker's pre-import preflight must call the gate with compute_subdirs=False so it # never imports model_config/transformers before the SSM kernels are installed. - src = (_BACKEND / "core" / "inference" / "worker.py").read_text() + src = (_BACKEND / "core" / "inference" / "worker.py").read_text(encoding = "utf-8") assert "compute_subdirs = False" in src @@ -506,7 +506,7 @@ def test_security_gates_run_before_ssm_install(): # The SSM install is name-based and can source-build native packages, so a malware / # blocked-code model must be refused first -- in both the pre-import path and _handle_load. import ast - tree = ast.parse((_BACKEND / "core" / "inference" / "worker.py").read_text()) + tree = ast.parse((_BACKEND / "core" / "inference" / "worker.py").read_text(encoding = "utf-8")) for fn in ("run_inference_process", "_handle_load"): gates = _call_linenos(tree, fn, "_run_security_gates") ssm = _call_linenos(tree, fn, "_ensure_ssm_kernels") diff --git a/studio/backend/tests/test_studio_api.py b/studio/backend/tests/test_studio_api.py index 087c00b648..13dfccde20 100644 --- a/studio/backend/tests/test_studio_api.py +++ b/studio/backend/tests/test_studio_api.py @@ -403,9 +403,9 @@ def test_openai_tools_stream(base_url: str, api_key: str): ) assert status == 200, f"Expected 200, got {status}" assert len(chunks) > 0, "No SSE chunks received" - assert _final_finish_reason(chunks) == "tool_calls", ( - f"Expected final finish_reason='tool_calls', got " f"{_final_finish_reason(chunks)!r}" - ) + assert ( + _final_finish_reason(chunks) == "tool_calls" + ), f"Expected final finish_reason='tool_calls', got {_final_finish_reason(chunks)!r}" assembled = _collect_streamed_tool_calls(chunks) assert len(assembled) >= 1, "No tool_calls reassembled from stream" first = assembled[0] @@ -486,16 +486,16 @@ def test_openai_sdk_tool_calling(base_url: str, api_key: str): tool_choice = "required", stream = False, ) - assert resp.choices[0].finish_reason == "tool_calls", ( - f"Expected finish_reason='tool_calls', got " f"{resp.choices[0].finish_reason!r}" - ) + assert ( + resp.choices[0].finish_reason == "tool_calls" + ), f"Expected finish_reason='tool_calls', got {resp.choices[0].finish_reason!r}" tool_calls = resp.choices[0].message.tool_calls assert tool_calls and len(tool_calls) >= 1, "No tool_calls from SDK" tc = tool_calls[0] assert tc.function.name == "get_weather" parsed = json.loads(tc.function.arguments) assert "city" in parsed - print(f" PASS openai SDK tool calling: " f"tool={tc.function.name}, args={parsed}") + print(f" PASS openai SDK tool calling: tool={tc.function.name}, args={parsed}") def test_invalid_key_rejected(base_url: str): @@ -783,12 +783,17 @@ def _start_server(model: str, variant: str | None) -> tuple[subprocess.Popen, st cmd.extend(["--gguf-variant", variant]) LOG_FILE.parent.mkdir(parents = True, exist_ok = True) - log_fh = open(LOG_FILE, "w") + log_fh = open(LOG_FILE, "w", encoding = "utf-8") + # The child writes to this descriptor itself, so the parent's encoding does + # not transcode anything: tell the child to emit utf-8 or the reads below + # decode its locale bytes as utf-8 and raise on the first non-ASCII glyph. + child_env = {**os.environ, "PYTHONIOENCODING": "utf-8", "PYTHONUTF8": "1"} proc = subprocess.Popen( cmd, stdout = log_fh, stderr = subprocess.STDOUT, preexec_fn = os.setsid, + env = child_env, ) # Wait for the banner containing the API key @@ -798,16 +803,16 @@ def _start_server(model: str, variant: str | None) -> tuple[subprocess.Popen, st time.sleep(2) if proc.poll() is not None: log_fh.flush() - log_text = LOG_FILE.read_text() + log_text = LOG_FILE.read_text(encoding = "utf-8") raise RuntimeError(f"Server exited early (code {proc.returncode}):\n{log_text[-2000:]}") - log_text = LOG_FILE.read_text() + log_text = LOG_FILE.read_text(encoding = "utf-8") m = re.search(r"API Key:\s+(sk-unsloth-[a-f0-9]+)", log_text) if m: api_key = m.group(1) break if not api_key: - log_text = LOG_FILE.read_text() + log_text = LOG_FILE.read_text(encoding = "utf-8") _kill_server(proc) raise RuntimeError(f"Timed out waiting for API key in server output:\n{log_text[-2000:]}") diff --git a/studio/backend/tests/test_tool_call_parser_strict.py b/studio/backend/tests/test_tool_call_parser_strict.py index 02f63c41a2..0bf627e8aa 100644 --- a/studio/backend/tests/test_tool_call_parser_strict.py +++ b/studio/backend/tests/test_tool_call_parser_strict.py @@ -64,9 +64,7 @@ class TestFunctionStyleTrailingText: # The real closing is the last one; the literal inside # the code argument must survive (rfind, not the first match). text = ( - "" - 'print("")' - " all done" + 'print("") all done' ) call = _only(text) assert call == {"name": "python", "arguments": {"code": 'print("")'}} @@ -146,9 +144,7 @@ class TestParityWithJsonStyle: class TestGemmaNativeStyle: def test_closed_native_call_with_trailing_prose_is_accepted(self): - text = ( - '<|tool_call>call:terminal{command:"ls -la",workdir:"."}' " running it now" - ) + text = '<|tool_call>call:terminal{command:"ls -la",workdir:"."} running it now' calls = parse_tool_calls_from_text(text, allow_incomplete = False) assert len(calls) == 1 assert calls[0]["function"]["name"] == "terminal" @@ -792,7 +788,7 @@ def test_tool_call_parser_declares_future_annotations_for_py39_import(): from pathlib import Path src = ( Path(__file__).resolve().parent.parent / "core" / "inference" / "tool_call_parser.py" - ).read_text() + ).read_text(encoding = "utf-8") assert "from __future__ import annotations" in src @@ -1069,8 +1065,7 @@ class TestBareJsonOuterOverXmlLiteral: def test_bare_json_code_arg_quoting_function_xml(self): text = ( - '{"name": "python", "arguments": ' - '{"code": "run() # ls"}}' + '{"name": "python", "arguments": {"code": "run() # ls"}}' ) calls = parse_tool_calls_from_text(text, enabled_tool_names = {"python"}) assert [c["function"]["name"] for c in calls] == ["python"] @@ -1300,8 +1295,7 @@ class TestLeadingWrapperlessGemmaOverEmbeddedMarkers: def test_leading_gemma_wins_over_quoted_xml_literal(self): text = ( - 'call:web_search{query:"explain ' - '{"name":"evil","arguments":{}}"}' + 'call:web_search{query:"explain {"name":"evil","arguments":{}}"}' ) calls = parse_tool_calls_from_text(text, enabled_tool_names = {"web_search", "evil"}) assert [c["function"]["name"] for c in calls] == ["web_search"] diff --git a/studio/backend/tests/test_tool_xml_strip.py b/studio/backend/tests/test_tool_xml_strip.py index f7792a2a71..941d9d044a 100644 --- a/studio/backend/tests/test_tool_xml_strip.py +++ b/studio/backend/tests/test_tool_xml_strip.py @@ -21,7 +21,7 @@ if _BACKEND_DIR not in sys.path: # Extract the regex from source (routes module needs heavy stubbing to import). import re as _re -_src = (Path(_BACKEND_DIR) / "routes" / "inference.py").read_text() +_src = (Path(_BACKEND_DIR) / "routes" / "inference.py").read_text(encoding = "utf-8") _m = _re.search(r"_TOOL_XML_RE = _re\.compile\((.*?)\n\)", _src, _re.DOTALL) assert _m, "could not extract _TOOL_XML_RE source" # The lazy ``(.*?)\n\)`` could grab a shorter expression if an arm is ever wrapped; diff --git a/studio/backend/tests/test_tp_vision_regression.py b/studio/backend/tests/test_tp_vision_regression.py index d1372ca415..5dfc38f9af 100644 --- a/studio/backend/tests/test_tp_vision_regression.py +++ b/studio/backend/tests/test_tp_vision_regression.py @@ -450,7 +450,7 @@ def test_fallback_hint_uses_effective_tensor_request_not_just_toggle(): """Tensor intent keys off _effective_tensor_parallel (toggle + extras + env), not just the toggle, so extra/env-driven tensor users keep multi-GPU (#6659).""" route = Path(_BACKEND_DIR) / "routes" / "inference.py" - src = route.read_text() + src = route.read_text(encoding = "utf-8") idx = src.find("_tensor_intent_overall = _effective_tensor_parallel(") assert idx != -1, "the GGUF load closure must compute tensor intent" block = src[idx : idx + 300] @@ -482,7 +482,7 @@ def test_preserved_fallback_carried_across_non_drop_reload(): gated on the same model loaded, so a ctx-only reload keeps multi-GPU but a model switch / explicit drop doesn't inherit it (#6659).""" route = Path(_BACKEND_DIR) / "routes" / "inference.py" - src = route.read_text() + src = route.read_text(encoding = "utf-8") idx = src.find("_tensor_intent_overall = _effective_tensor_parallel(") assert idx != -1 block = src[idx : idx + 400] @@ -499,7 +499,7 @@ def test_same_model_guard_checks_path_and_variant(): repo), so a reload keeps the carry-forward and a different variant doesn't inherit the prior one's preserved tensor intent (#6659).""" route = Path(_BACKEND_DIR) / "routes" / "inference.py" - src = route.read_text() + src = route.read_text(encoding = "utf-8") idx = src.find("_same_model_loaded = (") assert idx != -1 block = src[idx : idx + 1300] @@ -748,7 +748,7 @@ def test_explicit_tensor_drop_uses_shared_helper_in_both_readers(): _is_explicit_tensor_drop, so they agree on what counts as a drop -- a reload for an unrelated extra still carries the preserved intent rather than collapsing to one GPU (Codex #6659).""" - src = (Path(_BACKEND_DIR) / "routes" / "inference.py").read_text() + src = (Path(_BACKEND_DIR) / "routes" / "inference.py").read_text(encoding = "utf-8") # Dedup reader (the preserved-fallback reload guard). assert "layer_preserves_tensor_intent and _is_explicit_tensor_drop(request)" in src # Load carry-forward reader feeds the same decision into the carry-forward. diff --git a/studio/backend/tests/test_training_raw_support.py b/studio/backend/tests/test_training_raw_support.py index fb3cffc91e..49281605e6 100644 --- a/studio/backend/tests/test_training_raw_support.py +++ b/studio/backend/tests/test_training_raw_support.py @@ -163,13 +163,13 @@ class TestTrainingRawSupport(unittest.TestCase): def test_route_forwards_all_grad_clipping_fields(self): # The HTTP route builds the config dict by hand; a schema field that # is not forwarded here is silently dropped for REST callers. - source = (_BACKEND_ROOT / "routes" / "training.py").read_text() + source = (_BACKEND_ROOT / "routes" / "training.py").read_text(encoding = "utf-8") self.assertIn('"max_grad_norm": request.max_grad_norm', source) self.assertIn('"max_grad_value": request.max_grad_value', source) self.assertIn('"max_grad_leaf_norm": request.max_grad_leaf_norm', source) def test_mlx_worker_falls_back_init_seeds_to_random_seed(self): - source = (_BACKEND_ROOT / "core" / "training" / "worker.py").read_text() + source = (_BACKEND_ROOT / "core" / "training" / "worker.py").read_text(encoding = "utf-8") # random_seed itself is normalized first so explicit None coming # from a raw / backend caller does not propagate through the chain. @@ -198,7 +198,7 @@ class TestTrainingRawSupport(unittest.TestCase): self.assertIn("seed = random_seed,", source) def test_mlx_worker_preserves_null_max_grad_value_for_trainer_default(self): - source = (_BACKEND_ROOT / "core" / "training" / "worker.py").read_text() + source = (_BACKEND_ROOT / "core" / "training" / "worker.py").read_text(encoding = "utf-8") # None must survive to the MLX trainer so it picks its own runtime # default, and any other value must coerce to float without @@ -251,7 +251,7 @@ class TestTrainingRawSupport(unittest.TestCase): # unsloth-zoo update. Until that floor is in place, the # worker must gate them so releases that predate those fields can # still construct MLXTrainingConfig without TypeError. - source = (_BACKEND_ROOT / "core" / "training" / "worker.py").read_text() + source = (_BACKEND_ROOT / "core" / "training" / "worker.py").read_text(encoding = "utf-8") self.assertIn( 'getattr(MLXTrainingConfig, "__dataclass_fields__", {})', diff --git a/studio/backend/tests/test_transformers_version.py b/studio/backend/tests/test_transformers_version.py index acb2ec449b..7926ace1d3 100644 --- a/studio/backend/tests/test_transformers_version.py +++ b/studio/backend/tests/test_transformers_version.py @@ -2672,7 +2672,7 @@ class TestLatestTierForces16Bit: def _read(self, rel): backend_dir = Path(__file__).resolve().parent.parent - return (backend_dir / rel).read_text() + return (backend_dir / rel).read_text(encoding = "utf-8") def test_worker_guard_present(self): src = self._read("core/inference/worker.py") diff --git a/studio/backend/tests/test_yaml_trust_remote_code_removed.py b/studio/backend/tests/test_yaml_trust_remote_code_removed.py index 9578f08420..fa313cf0fa 100644 --- a/studio/backend/tests/test_yaml_trust_remote_code_removed.py +++ b/studio/backend/tests/test_yaml_trust_remote_code_removed.py @@ -19,7 +19,7 @@ _MODEL_DEFAULTS = _CONFIGS / "model_defaults" def test_no_model_default_yaml_sets_trust_remote_code(): offenders = [] for f in _MODEL_DEFAULTS.rglob("*.yaml"): - doc = yaml.safe_load(f.read_text()) or {} + doc = yaml.safe_load(f.read_text(encoding = "utf-8")) or {} if not isinstance(doc, dict): continue for section, body in doc.items(): @@ -37,7 +37,7 @@ def test_no_model_default_yaml_has_empty_or_none_section(): # A bare `inference:` header (no keys) parses to None and crashes the .get() loaders. offenders = [] for f in _MODEL_DEFAULTS.rglob("*.yaml"): - doc = yaml.safe_load(f.read_text()) + doc = yaml.safe_load(f.read_text(encoding = "utf-8")) if not isinstance(doc, dict): offenders.append(f"{f.relative_to(_CONFIGS)} (not a mapping)") continue @@ -96,7 +96,7 @@ def test_all_model_yamls_load_for_training_and_inference(): def test_base_templates_have_no_trust_remote_code(): for name in ("full_finetune.yaml", "lora_text.yaml", "vision_lora.yaml"): - doc = yaml.safe_load((_CONFIGS / name).read_text()) or {} + doc = yaml.safe_load((_CONFIGS / name).read_text(encoding = "utf-8")) or {} flat = yaml.safe_dump(doc) assert "trust_remote_code" not in flat, f"{name} should not set trust_remote_code" diff --git a/tests/python/test_cpo_processor_text_tokenizer.py b/tests/python/test_cpo_processor_text_tokenizer.py index 69316e042d..9440cba28a 100644 --- a/tests/python/test_cpo_processor_text_tokenizer.py +++ b/tests/python/test_cpo_processor_text_tokenizer.py @@ -40,7 +40,7 @@ def _registrations(source): def test_cpo_registration_matches_orpo(): - regs = _registrations(open(RL_PATH).read()) + regs = _registrations(open(RL_PATH, encoding = "utf-8").read()) shared = {"orpo_trainer_text_tokenizer", "orpo_trainer_processor_pad_token"} assert shared <= set(regs.get("orpo_trainer", [])) assert shared <= set(regs.get("cpo_trainer", [])) @@ -48,7 +48,7 @@ def test_cpo_registration_matches_orpo(): def _load_pad_rewriter(): """Exec orpo_trainer_processor_pad_token (+ _PAD_FALLBACK) without importing unsloth.""" - tree = ast.parse(open(RL_PATH).read()) + tree = ast.parse(open(RL_PATH, encoding = "utf-8").read()) nodes = [] for n in tree.body: if isinstance(n, ast.Assign) and any( diff --git a/tests/python/test_dpo_vision_processor_passthrough.py b/tests/python/test_dpo_vision_processor_passthrough.py index a320cab935..f9f8cee24f 100644 --- a/tests/python/test_dpo_vision_processor_passthrough.py +++ b/tests/python/test_dpo_vision_processor_passthrough.py @@ -11,7 +11,7 @@ RL_PATH = os.path.join(REPO_ROOT, "unsloth", "models", "rl_replacements.py") def _load_helpers(): - src = open(RL_PATH).read() + src = open(RL_PATH, encoding = "utf-8").read() tree = ast.parse(src) import torch as _torch diff --git a/tests/python/test_e2e_no_torch_sandbox.py b/tests/python/test_e2e_no_torch_sandbox.py index bb61af462d..3e46f4145e 100644 --- a/tests/python/test_e2e_no_torch_sandbox.py +++ b/tests/python/test_e2e_no_torch_sandbox.py @@ -193,7 +193,7 @@ class TestBeforeAfterImportChain: mm = types.ModuleType('model_mappings') mm.MODEL_TO_TEMPLATE_MAPPER = {{}} sys.modules['model_mappings'] = mm - source = open({str(before_file)!r}).read() + source = open({str(before_file)!r}, encoding = "utf-8").read() source = source.replace('from .format_detection import', 'from format_detection import') source = source.replace('from .model_mappings import', 'from model_mappings import') exec(source) @@ -215,7 +215,7 @@ class TestBeforeAfterImportChain: loggers = types.ModuleType('loggers') loggers.get_logger = lambda n: None sys.modules['loggers'] = loggers - exec(open({str(before_file)!r}).read()) + exec(open({str(before_file)!r}, encoding = "utf-8").read()) """) result = _run_in_sandbox(no_torch_venv, code) assert result.returncode != 0, "BEFORE data_collators.py should crash without torch" @@ -284,7 +284,7 @@ class TestBeforeAfterImportChain: it = types.ModuleType('iterable') it.is_streaming_dataset = lambda *a, **k: False sys.modules['iterable'] = it - source = open({str(CHAT_TEMPLATES)!r}).read() + source = open({str(CHAT_TEMPLATES)!r}, encoding = "utf-8").read() source = source.replace('from .format_detection import', 'from format_detection import') source = source.replace('from .model_mappings import', 'from model_mappings import') source = source.replace('from .iterable import', 'from iterable import') @@ -304,7 +304,7 @@ class TestBeforeAfterImportChain: loggers = types.ModuleType('loggers') loggers.get_logger = lambda n: None sys.modules['loggers'] = loggers - exec(open({str(DATA_COLLATORS)!r}).read()) + exec(open({str(DATA_COLLATORS)!r}, encoding = "utf-8").read()) print("OK") """) result = _run_in_sandbox(no_torch_venv, code) @@ -382,7 +382,7 @@ class TestDataclassInstantiation: loggers = types.ModuleType('loggers') loggers.get_logger = lambda n: None sys.modules['loggers'] = loggers - exec(open({str(DATA_COLLATORS)!r}).read()) + exec(open({str(DATA_COLLATORS)!r}, encoding = "utf-8").read()) obj = DataCollatorSpeechSeq2SeqWithPadding(processor=None) assert obj.processor is None print("OK") @@ -397,7 +397,7 @@ class TestDataclassInstantiation: loggers = types.ModuleType('loggers') loggers.get_logger = lambda n: None sys.modules['loggers'] = loggers - exec(open({str(DATA_COLLATORS)!r}).read()) + exec(open({str(DATA_COLLATORS)!r}, encoding = "utf-8").read()) obj = DeepSeekOCRDataCollator(processor=None) assert obj.processor is None assert obj.max_length == 2048 @@ -414,7 +414,7 @@ class TestDataclassInstantiation: loggers = types.ModuleType('loggers') loggers.get_logger = lambda n: None sys.modules['loggers'] = loggers - exec(open({str(DATA_COLLATORS)!r}).read()) + exec(open({str(DATA_COLLATORS)!r}, encoding = "utf-8").read()) obj = VLMDataCollator(processor=None) assert obj.processor is None assert obj.max_length == 2048 @@ -441,7 +441,7 @@ class TestDataclassInstantiation: it.is_streaming_dataset = lambda *a, **k: False sys.modules['iterable'] = it ns = {{}} - source = open({str(CHAT_TEMPLATES)!r}).read() + source = open({str(CHAT_TEMPLATES)!r}, encoding = "utf-8").read() source = source.replace('from .format_detection import', 'from format_detection import') source = source.replace('from .model_mappings import', 'from model_mappings import') source = source.replace('from .iterable import', 'from iterable import') @@ -473,7 +473,7 @@ class TestEdgeCasesBrokenTorch: code = textwrap.dedent(f"""\ import sys sys.path.insert(0, {str(sandbox_dir)!r}) - exec(open({str(sandbox_dir / 'data_collators.py')!r}).read()) + exec(open({str(sandbox_dir / 'data_collators.py')!r}, encoding = "utf-8").read()) obj = DataCollatorSpeechSeq2SeqWithPadding(processor=None) print("OK: data_collators works despite broken torch on sys.path") """) @@ -495,7 +495,7 @@ class TestEdgeCasesBrokenTorch: code = textwrap.dedent(f"""\ import sys sys.path.insert(0, {str(sandbox_dir)!r}) - source = open({str(HARDWARE_PY)!r}).read() + source = open({str(HARDWARE_PY)!r}, encoding = "utf-8").read() ns = {{'__name__': '__test__'}} exec(source, ns) result = ns['detect_hardware']() @@ -530,7 +530,7 @@ class TestEdgeCasesBrokenTorch: code = textwrap.dedent(f"""\ import sys sys.path.insert(0, {str(sandbox_dir)!r}) - source = open({str(HARDWARE_PY)!r}).read() + source = open({str(HARDWARE_PY)!r}, encoding = "utf-8").read() ns = {{'__name__': '__test__'}} exec(source, ns) result = ns['detect_hardware']() @@ -559,7 +559,7 @@ class TestEdgeCasesBrokenTorch: sys.modules['iterable'] = it ns = {{}} - source = open({str(CHAT_TEMPLATES)!r}).read() + source = open({str(CHAT_TEMPLATES)!r}, encoding = "utf-8").read() source = source.replace('from .format_detection import', 'from format_detection import') source = source.replace('from .model_mappings import', 'from model_mappings import') source = source.replace('from .iterable import', 'from iterable import') @@ -604,7 +604,7 @@ class TestHardwareDetectionNoTorch: code = textwrap.dedent(f"""\ import sys sys.path.insert(0, {str(sandbox_dir)!r}) - source = open({str(HARDWARE_PY)!r}).read() + source = open({str(HARDWARE_PY)!r}, encoding = "utf-8").read() ns = {{'__name__': '__test__'}} exec(source, ns) device = ns['detect_hardware']() @@ -624,7 +624,7 @@ class TestHardwareDetectionNoTorch: code = textwrap.dedent(f"""\ import sys sys.path.insert(0, {str(sandbox_dir)!r}) - source = open({str(HARDWARE_PY)!r}).read() + source = open({str(HARDWARE_PY)!r}, encoding = "utf-8").read() ns = {{'__name__': '__test__'}} exec(source, ns) versions = ns['get_package_versions']() @@ -651,7 +651,7 @@ class TestHardwareDetectionNoTorch: code = textwrap.dedent(f"""\ import sys sys.path.insert(0, {str(sandbox_dir)!r}) - source = open({str(hw_sandbox / 'hardware.py')!r}).read() + source = open({str(hw_sandbox / 'hardware.py')!r}, encoding = "utf-8").read() ns = {{'__name__': '__test__'}} exec(source, ns) assert callable(ns['detect_hardware']) diff --git a/tests/python/test_fast_language_model_text_only.py b/tests/python/test_fast_language_model_text_only.py index fcdeb49bc3..08e5cdf0dc 100644 --- a/tests/python/test_fast_language_model_text_only.py +++ b/tests/python/test_fast_language_model_text_only.py @@ -14,7 +14,7 @@ UTILS_PATH = REPO_ROOT / "unsloth" / "models" / "_utils.py" def _source(path): - return path.read_text() + return path.read_text(encoding = "utf-8") def _class_method(tree, class_name, method_name): diff --git a/tests/python/test_fast_model_config_passthrough.py b/tests/python/test_fast_model_config_passthrough.py index b2ba3d2eef..6ab941478e 100644 --- a/tests/python/test_fast_model_config_passthrough.py +++ b/tests/python/test_fast_model_config_passthrough.py @@ -12,7 +12,7 @@ LLAMA_PATH = REPO_ROOT / "unsloth" / "models" / "llama.py" def _source(path): - return path.read_text() + return path.read_text(encoding = "utf-8") def _class_method(tree, class_name, method_name): diff --git a/tests/python/test_gpu_init_ldconfig_guard.py b/tests/python/test_gpu_init_ldconfig_guard.py index 248bb84faa..986dfcec8b 100644 --- a/tests/python/test_gpu_init_ldconfig_guard.py +++ b/tests/python/test_gpu_init_ldconfig_guard.py @@ -17,13 +17,13 @@ def _find_geteuid_guard(tree: ast.AST): def test_gpu_init_has_geteuid_guard(): - tree = ast.parse(GPU_INIT.read_text()) + tree = ast.parse(GPU_INIT.read_text(encoding = "utf-8")) guard = _find_geteuid_guard(tree) assert guard is not None, "_gpu_init.py must guard ldconfig recovery on os.geteuid()" def test_ldconfig_calls_only_inside_geteuid_guard(): - src = GPU_INIT.read_text() + src = GPU_INIT.read_text(encoding = "utf-8") tree = ast.parse(src) guard = _find_geteuid_guard(tree) assert guard is not None @@ -39,6 +39,6 @@ def test_ldconfig_calls_only_inside_geteuid_guard(): def test_non_root_branch_warns_when_bnb_present(): - src = GPU_INIT.read_text() + src = GPU_INIT.read_text(encoding = "utf-8") assert "elif bnb is not None" in src assert "sudo ldconfig" in src diff --git a/tests/python/test_grpo_ddp_model_config.py b/tests/python/test_grpo_ddp_model_config.py index 5af31f65b8..23614d3add 100644 --- a/tests/python/test_grpo_ddp_model_config.py +++ b/tests/python/test_grpo_ddp_model_config.py @@ -9,7 +9,7 @@ SOURCE_PATH = os.path.join(REPO_ROOT, "unsloth", "models", "rl_replacements.py") def _read_source() -> str: - with open(SOURCE_PATH, "r") as fh: + with open(SOURCE_PATH, "r", encoding = "utf-8") as fh: return fh.read() diff --git a/tests/python/test_orpo_processor_text_tokenizer.py b/tests/python/test_orpo_processor_text_tokenizer.py index b507a9e808..84bfe60bb0 100644 --- a/tests/python/test_orpo_processor_text_tokenizer.py +++ b/tests/python/test_orpo_processor_text_tokenizer.py @@ -10,7 +10,7 @@ RL_PATH = os.path.join(REPO_ROOT, "unsloth", "models", "rl_replacements.py") def _load_orpo_rewriter(name = "orpo_trainer_text_tokenizer"): - src = open(RL_PATH).read() + src = open(RL_PATH, encoding = "utf-8").read() tree = ast.parse(src) ns = {"re": re} # Materialise sibling module-level _-prefixed assignments the rewriter may reference. diff --git a/tests/python/test_pad_token_fix.py b/tests/python/test_pad_token_fix.py index 5f2a29a323..c19c6969ce 100644 --- a/tests/python/test_pad_token_fix.py +++ b/tests/python/test_pad_token_fix.py @@ -21,7 +21,7 @@ WANTED = { def _load_pad_helpers(): """Exec only the pad-token helpers with a stub logger (no heavy imports).""" - tree = ast.parse(open(TOK_PATH).read()) + tree = ast.parse(open(TOK_PATH, encoding = "utf-8").read()) nodes = [] for node in tree.body: if isinstance(node, ast.Assign): diff --git a/tests/python/test_studio_import_no_torch.py b/tests/python/test_studio_import_no_torch.py index f551519de9..48b62fd99d 100644 --- a/tests/python/test_studio_import_no_torch.py +++ b/tests/python/test_studio_import_no_torch.py @@ -148,7 +148,7 @@ class TestDataCollatorsNoTorchVenv: loggers = types.ModuleType('loggers') loggers.get_logger = lambda n: None sys.modules['loggers'] = loggers - exec(open({str(DATA_COLLATORS)!r}).read()) + exec(open({str(DATA_COLLATORS)!r}, encoding = "utf-8").read()) print("OK: exec succeeded") """) result = subprocess.run( @@ -168,7 +168,7 @@ class TestDataCollatorsNoTorchVenv: loggers = types.ModuleType('loggers') loggers.get_logger = lambda n: None sys.modules['loggers'] = loggers - exec(open({str(DATA_COLLATORS)!r}).read()) + exec(open({str(DATA_COLLATORS)!r}, encoding = "utf-8").read()) obj = DataCollatorSpeechSeq2SeqWithPadding(processor=None) assert obj.processor is None, "processor should be None" print("OK: DataCollatorSpeechSeq2SeqWithPadding instantiated") @@ -190,7 +190,7 @@ class TestDataCollatorsNoTorchVenv: loggers = types.ModuleType('loggers') loggers.get_logger = lambda n: None sys.modules['loggers'] = loggers - exec(open({str(DATA_COLLATORS)!r}).read()) + exec(open({str(DATA_COLLATORS)!r}, encoding = "utf-8").read()) obj = DeepSeekOCRDataCollator(processor=None) assert obj.processor is None, "processor should be None" assert obj.max_length == 2048, "default max_length should be 2048" @@ -212,7 +212,7 @@ class TestDataCollatorsNoTorchVenv: loggers = types.ModuleType('loggers') loggers.get_logger = lambda n: None sys.modules['loggers'] = loggers - exec(open({str(DATA_COLLATORS)!r}).read()) + exec(open({str(DATA_COLLATORS)!r}, encoding = "utf-8").read()) obj = VLMDataCollator(processor=None) assert obj.processor is None assert obj.mask_input_tokens is True, "default mask_input_tokens should be True" @@ -259,7 +259,7 @@ class TestChatTemplatesNoTorchVenv: sys.modules['iterable'] = iterable # Read and transform the source: replace relative imports with absolute - source = open({str(CHAT_TEMPLATES)!r}).read() + source = open({str(CHAT_TEMPLATES)!r}, encoding = "utf-8").read() source = source.replace('from .format_detection import', 'from format_detection import') source = source.replace('from .model_mappings import', 'from model_mappings import') source = source.replace('from .iterable import', 'from iterable import') @@ -305,7 +305,7 @@ class TestChatTemplatesNoTorchVenv: sys.modules['iterable'] = iterable ns = {{}} - source = open({str(CHAT_TEMPLATES)!r}).read() + source = open({str(CHAT_TEMPLATES)!r}, encoding = "utf-8").read() source = source.replace('from .format_detection import', 'from format_detection import') source = source.replace('from .model_mappings import', 'from model_mappings import') source = source.replace('from .iterable import', 'from iterable import') @@ -402,7 +402,7 @@ class TestFormatConversionNoTorchVenv: sys.modules['utils.hardware'] = hardware_mod # Read and exec format_conversion.py - source = open({str(FORMAT_CONVERSION)!r}).read() + source = open({str(FORMAT_CONVERSION)!r}, encoding = "utf-8").read() source = source.replace('from .format_detection import', 'from format_detection import') source = source.replace('from .iterable import', 'from iterable import') ns = {{'__name__': '__test__'}} @@ -463,7 +463,7 @@ class TestFormatConversionNoTorchVenv: sys.modules['utils'] = utils_mod sys.modules['utils.hardware'] = hardware_mod - source = open({str(FORMAT_CONVERSION)!r}).read() + source = open({str(FORMAT_CONVERSION)!r}, encoding = "utf-8").read() source = source.replace('from .format_detection import', 'from format_detection import') source = source.replace('from .iterable import', 'from iterable import') ns = {{'__name__': '__test__'}} @@ -517,7 +517,7 @@ class TestNegativeControls: loggers = types.ModuleType('loggers') loggers.get_logger = lambda n: None sys.modules['loggers'] = loggers - exec(open({temp_file!r}).read()) + exec(open({temp_file!r}, encoding = "utf-8").read()) """) result = subprocess.run( [no_torch_venv, "-c", code], diff --git a/tests/python/test_v100_fullft_precision.py b/tests/python/test_v100_fullft_precision.py index c8ca769d45..49abcb4ad6 100644 --- a/tests/python/test_v100_fullft_precision.py +++ b/tests/python/test_v100_fullft_precision.py @@ -29,7 +29,7 @@ RL_PY = Path(__file__).resolve().parents[2] / "unsloth" / "models" / "rl.py" def _extract_mixed_precision_code() -> str: - lines = RL_PY.read_text().split("\n") + lines = RL_PY.read_text(encoding = "utf-8").split("\n") try: start = next(i for i, l in enumerate(lines) if "mixed_precision = (" in l) except StopIteration: diff --git a/tests/python/test_vision_lora_targeting.py b/tests/python/test_vision_lora_targeting.py index 0a27569efd..bed26aa297 100644 --- a/tests/python/test_vision_lora_targeting.py +++ b/tests/python/test_vision_lora_targeting.py @@ -37,7 +37,7 @@ def test_vlm_lora_regex_respects_language_only_with_explicit_targets(): def test_fast_vision_model_wraps_explicit_targets_when_layer_filters_are_used(): - source = Path("unsloth/models/vision.py").read_text() + source = Path("unsloth/models/vision.py").read_text(encoding = "utf-8") assert "target_modules = get_peft_regex(" in source assert "target_modules = list(target_modules)" in source diff --git a/tests/saving/test_fix_sentencepiece_gguf_robustness.py b/tests/saving/test_fix_sentencepiece_gguf_robustness.py index 9c61ca4067..2b9cc87a60 100644 --- a/tests/saving/test_fix_sentencepiece_gguf_robustness.py +++ b/tests/saving/test_fix_sentencepiece_gguf_robustness.py @@ -82,7 +82,7 @@ def test_entry_with_non_int_id_is_skipped(tmp_path): def test_save_py_except_clause_is_broad_exception(): - with open(_SAVE_PY) as f: + with open(_SAVE_PY, encoding = "utf-8") as f: tree = ast.parse(f.read()) for node in ast.walk(tree): if isinstance(node, ast.FunctionDef) and node.name == "unsloth_save_pretrained_gguf": @@ -102,7 +102,7 @@ def test_save_py_except_clause_is_broad_exception(): def test_tokenizer_utils_uses_import_protobuf_fallback_pattern(): - with open(_TOK_PY) as f: + with open(_TOK_PY, encoding = "utf-8") as f: src = f.read() tree = ast.parse(src) for node in ast.walk(tree): diff --git a/tests/security/test_scan_packages.py b/tests/security/test_scan_packages.py index 48e6da5f66..2608494b42 100644 --- a/tests/security/test_scan_packages.py +++ b/tests/security/test_scan_packages.py @@ -35,9 +35,11 @@ def test_fixture_bytes_are_deterministic(tmp_path): rebuild_dir = tmp_path / "rebuild" rebuild_dir.mkdir() # The build helper writes to its own dir; copy + patch HERE. - builder_src = (FIXTURES / "_build.py").read_text() + builder_src = (FIXTURES / "_build.py").read_text(encoding = "utf-8") rebuilt_helper = rebuild_dir / "_build.py" - rebuilt_helper.write_text(builder_src) + # builder_src came out of a checked-in file, so it carries whatever + # non-ASCII that file holds and cp1252 cannot encode it back out. + rebuilt_helper.write_text(builder_src, encoding = "utf-8") # Run with SOURCE_DATE_EPOCH=0 and HERE override via a shim. shim = rebuild_dir / "run.py" shim.write_text( @@ -1260,7 +1262,7 @@ def test_committed_baseline_suppresses_known_but_not_a_new_payload(): import json baseline_path = REPO_ROOT / "scripts" / "scan_packages_baseline.json" - entries = json.loads(baseline_path.read_text())["entries"] + entries = json.loads(baseline_path.read_text(encoding = "utf-8"))["entries"] target = next( e for e in entries @@ -1296,7 +1298,7 @@ def test_committed_baseline_entries_all_carry_evidence_hash(): import json baseline_path = REPO_ROOT / "scripts" / "scan_packages_baseline.json" - entries = json.loads(baseline_path.read_text())["entries"] + entries = json.loads(baseline_path.read_text(encoding = "utf-8"))["entries"] assert entries, "committed baseline should not be empty" missing = [ f"{e['package']}:{e['file']}:{e['check']}" for e in entries if not e.get("evidence_hash") diff --git a/tests/studio/install/test_llama_pr_force_and_source.py b/tests/studio/install/test_llama_pr_force_and_source.py index 4ff8c349c3..8d89660924 100644 --- a/tests/studio/install/test_llama_pr_force_and_source.py +++ b/tests/studio/install/test_llama_pr_force_and_source.py @@ -346,7 +346,7 @@ class TestSourcePatternsSh: @pytest.fixture(autouse = True) def _load_source(self): - self.content = SETUP_SH.read_text() + self.content = SETUP_SH.read_text(encoding = "utf-8") def test_has_default_pr_force(self): assert '_DEFAULT_LLAMA_PR_FORCE=""' in self.content @@ -412,7 +412,7 @@ class TestSourcePatternsPs1: @pytest.fixture(autouse = True) def _load_source(self): - self.content = SETUP_PS1.read_text() + self.content = SETUP_PS1.read_text(encoding = "utf-8") def test_has_default_pr_force(self): assert '$DefaultLlamaPrForce = ""' in self.content diff --git a/tests/studio/install/test_managed_node_runtime.py b/tests/studio/install/test_managed_node_runtime.py index 17c7e3e60f..251cb0ffcd 100644 --- a/tests/studio/install/test_managed_node_runtime.py +++ b/tests/studio/install/test_managed_node_runtime.py @@ -125,7 +125,7 @@ def test_resolve_falls_back_to_managed_when_no_system(monkeypatch, tmp_path): monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path)) managed = nr.managed_node_binary() managed.parent.mkdir(parents = True, exist_ok = True) - managed.write_text("#!/bin/sh\necho v24.17.0\n") + managed.write_text("#!/bin/sh\necho v24.17.0\n", encoding = "utf-8") monkeypatch.setattr(nr.shutil, "which", lambda name: None) monkeypatch.setattr(nr, "_node_version_ok", lambda exe: str(exe) == str(managed)) assert nr.resolve_node_executable() == str(managed) @@ -136,7 +136,7 @@ def test_resolve_prefers_managed_over_unsuitable_system(monkeypatch, tmp_path): monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path)) managed = nr.managed_node_binary() managed.parent.mkdir(parents = True, exist_ok = True) - managed.write_text("fake") + managed.write_text("fake", encoding = "utf-8") monkeypatch.setattr(nr.shutil, "which", lambda name: "/old/node") monkeypatch.setattr(nr, "_node_version_ok", lambda exe: str(exe) == str(managed)) assert nr.resolve_node_executable() == str(managed) @@ -167,7 +167,7 @@ def test_negative_result_is_not_cached(monkeypatch, tmp_path): managed = nr.managed_node_binary() managed.parent.mkdir(parents = True, exist_ok = True) - managed.write_text("now-installed") + managed.write_text("now-installed", encoding = "utf-8") monkeypatch.setattr(nr, "_node_version_ok", lambda exe: str(exe) == str(managed)) assert nr.resolve_node_executable() == str(managed) diff --git a/tests/studio/install/test_pr4562_bugfixes.py b/tests/studio/install/test_pr4562_bugfixes.py index 0d2b092924..4e555d76c6 100644 --- a/tests/studio/install/test_pr4562_bugfixes.py +++ b/tests/studio/install/test_pr4562_bugfixes.py @@ -617,7 +617,7 @@ class TestSourceCodePatterns: def test_setup_sh_no_rm_before_prereq_check(self): """rm -rf must appear AFTER cmake/git checks, not before.""" - content = SETUP_SH.read_text() + content = SETUP_SH.read_text(encoding = "utf-8") # Anchor on the source-build cmake check block. idx_block = content.find("command -v cmake") assert idx_block != -1 @@ -630,7 +630,7 @@ class TestSourceCodePatterns: def test_setup_sh_clone_uses_branch_tag(self): """git clone in source-build should use --branch via the clone args array.""" - content = SETUP_SH.read_text() + content = SETUP_SH.read_text(encoding = "utf-8") assert "_CLONE_ARGS=(git clone --depth 1)" in content assert ( '_CLONE_ARGS+=(--branch "$_RESOLVED_SOURCE_REF")' in content @@ -642,7 +642,7 @@ class TestSourceCodePatterns: def test_setup_sh_source_build_uses_helper_latest_tag_only(self): """Shell source fallback should only use helper latest-tag resolution.""" - content = SETUP_SH.read_text() + content = SETUP_SH.read_text(encoding = "utf-8") assert "--resolve-source-build" not in content assert "--resolve-install-tag" not in content assert '--resolve-llama-tag latest --published-repo "ggml-org/llama.cpp"' in content @@ -653,7 +653,7 @@ class TestSourceCodePatterns: def test_setup_sh_prebuilt_install_entrypoint(self): """Shell prebuilt path uses the helper install entrypoint, not the old releases-latest flow.""" - content = SETUP_SH.read_text() + content = SETUP_SH.read_text(encoding = "utf-8") assert "--resolve-install-tag" not in content assert "_HELPER_RELEASE_REPO}/releases/latest" not in content assert "ggml-org/llama.cpp/releases/latest" not in content @@ -663,7 +663,7 @@ class TestSourceCodePatterns: fork like every other host, so the release-repo decision is unconditional. Guards against a silent reintroduction of a ggml-org CPU routing branch. GPU usability detection (used for PyTorch / source decisions) must stay.""" - content = SETUP_SH.read_text() + content = SETUP_SH.read_text(encoding = "utf-8") assert '_HELPER_RELEASE_REPO="unslothai/llama.cpp"' in content assert '_HELPER_RELEASE_REPO="ggml-org/llama.cpp"' not in content # Usability gating (not routing) still distinguishes a hidden GPU. @@ -676,14 +676,14 @@ class TestSourceCodePatterns: def test_setup_sh_reports_installed_prebuilt_release(self): """Shell wrapper should report the installed prebuilt release from metadata.""" - content = SETUP_SH.read_text() + content = SETUP_SH.read_text(encoding = "utf-8") assert "UNSLOTH_PREBUILT_INFO.json" in content assert "installed release:" in content assert 'print_installed_llama_prebuilt_release "$LLAMA_CPP_DIR"' in content def test_setup_sh_macos_arm64_uses_metal_flags(self): """Apple Silicon source builds should explicitly enable Metal like upstream.""" - content = SETUP_SH.read_text() + content = SETUP_SH.read_text(encoding = "utf-8") assert "_IS_MACOS_ARM64=true" in content assert 'if [ "$_IS_MACOS_ARM64" = true ]; then' in content assert "-DGGML_METAL=ON" in content @@ -695,7 +695,7 @@ class TestSourceCodePatterns: def test_setup_sh_macos_metal_configure_has_cpu_fallback(self): """GPU configure/build failure retries a CPU build. Stays label-agnostic (PR #5826 generalised the Metal-only wording via $_FB_LABEL).""" - content = SETUP_SH.read_text() + content = SETUP_SH.read_text(encoding = "utf-8") assert "_TRY_METAL_CPU_FALLBACK=true" in content assert 'configure failed; retrying CPU build..." "$C_WARN"' in content assert 'build failed; retrying CPU build..." "$C_WARN"' in content @@ -714,7 +714,7 @@ class TestSourceCodePatterns: """PR #5826: a fresh CUDA toolkit's host-compiler whitelist lags distro gcc/clang (nvcc "#error -- unsupported GNU version"). setup.sh exports NVCC_PREPEND_FLAGS=-allow-unsupported-compiler via env, not CMAKE_ARGS (word-splitting safety).""" - content = SETUP_SH.read_text() + content = SETUP_SH.read_text(encoding = "utf-8") assert "-allow-unsupported-compiler" in content # Via NVCC_PREPEND_FLAGS (covers the configure-time probe too), not CMAKE_ARGS. assert "export NVCC_PREPEND_FLAGS=" in content @@ -726,7 +726,7 @@ class TestSourceCodePatterns: def test_setup_ps1_exports_allow_unsupported_compiler(self): """Windows parity for PR #5826: CUDA toolkit whitelist lags MSVC. setup.ps1 sets NVCC_PREPEND_FLAGS=-allow-unsupported-compiler in the CUDA branch via env, out of $CmakeArgs.""" - content = SETUP_PS1.read_text() + content = SETUP_PS1.read_text(encoding = "utf-8") assert "-allow-unsupported-compiler" in content # Via process env, not $CmakeArgs, so it reaches both the configure probe and `cmake --build`. assert "$env:NVCC_PREPEND_FLAGS" in content @@ -763,7 +763,7 @@ class TestSourceCodePatterns: def test_setup_sh_does_not_enable_metal_for_intel_macos(self): """Intel macOS should stay on the existing non-Metal path in this patch.""" - content = SETUP_SH.read_text() + content = SETUP_SH.read_text(encoding = "utf-8") assert 'if [ "$_IS_MACOS_ARM64" = true ]; then' in content assert ( 'Darwin" ] && { [ "$_HOST_MACHINE" = "arm64" ] || [ "$_HOST_MACHINE" = "aarch64" ]; }' @@ -778,20 +778,20 @@ class TestSourceCodePatterns: def test_setup_ps1_uses_checkout_b(self): """PS1 should use checkout -B, not checkout --force FETCH_HEAD.""" - content = SETUP_PS1.read_text() + content = SETUP_PS1.read_text(encoding = "utf-8") assert "checkout -B unsloth-llama-build" in content assert "checkout --force FETCH_HEAD" not in content def test_setup_ps1_clone_uses_branch_tag(self): """PS1 clone should use --branch with the resolved tag.""" - content = SETUP_PS1.read_text() + content = SETUP_PS1.read_text(encoding = "utf-8") assert "--branch" in content and "$ResolvedSourceRef" in content # The old commented-out clone line should be gone. assert "# git clone --depth 1 --branch" not in content def test_setup_ps1_no_git_pull(self): """PS1 should use fetch, not pull (which fails in detached HEAD).""" - content = SETUP_PS1.read_text() + content = SETUP_PS1.read_text(encoding = "utf-8") # No "git pull" in the source-build section (only valid on a branch). lines = content.splitlines() for i, line in enumerate(lines): @@ -800,18 +800,18 @@ class TestSourceCodePatterns: # Allowed elsewhere; fail only in the llama.cpp build section. context = "\n".join(lines[max(0, i - 5) : i + 5]) if "LlamaCppDir" in context: - pytest.fail(f"Found 'git pull' in llama.cpp build section at line {i+1}") + pytest.fail(f"Found 'git pull' in llama.cpp build section at line {i + 1}") def test_setup_ps1_prebuilt_install_entrypoint(self): """PS1 prebuilt path uses the helper install entrypoint, not the old releases-latest flow.""" - content = SETUP_PS1.read_text() + content = SETUP_PS1.read_text(encoding = "utf-8") assert "--resolve-install-tag" not in content assert "$HelperReleaseRepo/releases/latest" not in content assert "ggml-org/llama.cpp/releases/latest" not in content def test_setup_ps1_reports_installed_prebuilt_release(self): """PS1 wrapper should report the installed prebuilt release from metadata.""" - content = SETUP_PS1.read_text() + content = SETUP_PS1.read_text(encoding = "utf-8") assert "Get-InstalledLlamaPrebuiltRelease" in content assert "UNSLOTH_PREBUILT_INFO.json" in content assert "installed release:" in content @@ -822,7 +822,7 @@ class TestSourceCodePatterns: def test_setup_ps1_source_build_uses_helper_latest_tag_only(self): """PS1 source fallback should only use helper latest-tag resolution.""" - content = SETUP_PS1.read_text() + content = SETUP_PS1.read_text(encoding = "utf-8") assert "--resolve-source-build" not in content assert "--resolve-install-tag" not in content assert ( @@ -835,7 +835,7 @@ class TestSourceCodePatterns: def test_setup_ps1_prebuilt_install_disables_native_error_abort(self): """PS1 prebuilt install should not abort setup on helper stderr.""" - content = SETUP_PS1.read_text() + content = SETUP_PS1.read_text(encoding = "utf-8") install_idx = content.index("& python @prebuiltArgs 2>&1") block = content[max(0, install_idx - 800) : install_idx + 800] assert "$PSNativeCommandUseErrorActionPreference = $false" in block @@ -844,7 +844,7 @@ class TestSourceCodePatterns: def test_setup_ps1_helper_disables_error_action_abort(self): """Helper resolution should suppress terminating NativeCommandError on PS 5.1.""" - content = SETUP_PS1.read_text() + content = SETUP_PS1.read_text(encoding = "utf-8") helper_idx = content.index("function Invoke-LlamaHelper") block = content[helper_idx : helper_idx + 2200] assert "$previousErrorActionPreference = $ErrorActionPreference" in block @@ -853,19 +853,19 @@ class TestSourceCodePatterns: def test_setup_ps1_uses_local_tempfile_helper(self): """PS1 should not depend on New-TemporaryFile being available anywhere.""" - content = SETUP_PS1.read_text() + content = SETUP_PS1.read_text(encoding = "utf-8") assert "function New-UnslothTemporaryFile" in content assert "$resolveErrorLog = New-TemporaryFile" not in content def test_setup_ps1_find_nvcc_uses_version_sort_for_latest_toolkit(self): """The unconstrained nvcc fallback should not sort toolkit dirs lexicographically.""" - content = SETUP_PS1.read_text() + content = SETUP_PS1.read_text(encoding = "utf-8") assert "Sort-Object Name | Select-Object -Last 1" not in content assert "Sort-Object { [version]($_.Name -replace '^v','') } -Descending" in content def test_binary_env_linux_has_binary_parent(self): """The Linux branch of binary_env should include binary_path.parent.""" - content = MODULE_PATH.read_text() + content = MODULE_PATH.read_text(encoding = "utf-8") in_func = False in_linux = False found = False diff --git a/tests/studio/load_freeze/test_load_orchestrator.py b/tests/studio/load_freeze/test_load_orchestrator.py index 64503d0e05..a1f4caa309 100644 --- a/tests/studio/load_freeze/test_load_orchestrator.py +++ b/tests/studio/load_freeze/test_load_orchestrator.py @@ -441,7 +441,7 @@ def test_load_model_caches_audio_type_inside_serial_load_lock(): """Audio-type detection must run inside load_model under _serial_load_lock, else a concurrent /load can replace the backend mid-probe (review on #5669).""" f = _REPO_ROOT / "studio" / "backend" / "core" / "inference" / "llama_cpp.py" - text = f.read_text() + text = f.read_text(encoding = "utf-8") assert ( "with self._serial_load_lock" in text ), "LlamaCppBackend.load_model must hold self._serial_load_lock" @@ -462,7 +462,7 @@ def test_routes_inference_reads_cached_audio_type_not_calls_detect(): """routes/inference.py must read cached _audio_type/_is_audio, not call detect_audio_type / init_audio_codec directly (both moved into load_model).""" f = _REPO_ROOT / "studio" / "backend" / "routes" / "inference.py" - text = f.read_text() + text = f.read_text(encoding = "utf-8") assert "llama_backend.detect_audio_type(" not in text, ( "routes/inference.py should not call detect_audio_type directly; " "load_model already cached it under the lock." @@ -485,7 +485,7 @@ def test_no_other_async_route_calls_detect_audio_type_unwrapped(): # function helper is excluded below. pattern = re.compile(r"\b\w+\.detect_audio_type\s*\(") for path in routes_dir.rglob("*.py"): - for i, line in enumerate(path.read_text().splitlines(), start = 1): + for i, line in enumerate(path.read_text(encoding = "utf-8").splitlines(), start = 1): m = pattern.search(line) if not m: continue diff --git a/tests/studio/playwright_chat_ime_i18n.py b/tests/studio/playwright_chat_ime_i18n.py index 5bebf0a9e8..7883df16be 100644 --- a/tests/studio/playwright_chat_ime_i18n.py +++ b/tests/studio/playwright_chat_ime_i18n.py @@ -241,10 +241,12 @@ with sync_playwright() as p: # Source-level guard: grep the unmounted edit/compare composers' JSX for dir="auto". _repo_root = Path(__file__).resolve().parents[2] - _thread_src = ( - _repo_root / "studio/frontend/src/components/assistant-ui/thread.tsx" - ).read_text() - _shared_src = (_repo_root / "studio/frontend/src/features/chat/shared-composer.tsx").read_text() + _thread_src = (_repo_root / "studio/frontend/src/components/assistant-ui/thread.tsx").read_text( + encoding = "utf-8" + ) + _shared_src = (_repo_root / "studio/frontend/src/features/chat/shared-composer.tsx").read_text( + encoding = "utf-8" + ) _edit_idx = _thread_src.find("aui-edit-composer-input") if _edit_idx == -1 or 'dir="auto"' not in _thread_src[_edit_idx : _edit_idx + 600]: soft_fail('edit composer source is missing dir="auto"') diff --git a/tests/studio/playwright_chat_ui.py b/tests/studio/playwright_chat_ui.py index a06e559100..b182e66f01 100644 --- a/tests/studio/playwright_chat_ui.py +++ b/tests/studio/playwright_chat_ui.py @@ -98,7 +98,7 @@ def expected_default_model(): / "defaults.py" ) try: - tree = ast.parse(defaults_path.read_text()) + tree = ast.parse(defaults_path.read_text(encoding = "utf-8")) except Exception as exc: fail(f"could not read {defaults_path}: {exc}") models = None diff --git a/tests/studio/studio_api_smoke.py b/tests/studio/studio_api_smoke.py index d30bd11dca..ce55c06223 100644 --- a/tests/studio/studio_api_smoke.py +++ b/tests/studio/studio_api_smoke.py @@ -142,7 +142,7 @@ except Exception as exc: # GET / cross-origin must NOT leak the bootstrap password in the served HTML. boot_path = AUTH_DIR / ".bootstrap_password" if boot_path.exists(): - bootstrap_pw = boot_path.read_text().strip() + bootstrap_pw = boot_path.read_text(encoding = "utf-8").strip() if bootstrap_pw: req = urllib.request.Request( f"{BASE}/", diff --git a/tests/studio/test_auth_form_input_count.py b/tests/studio/test_auth_form_input_count.py index 75e6cfd1fb..aa7975cf10 100644 --- a/tests/studio/test_auth_form_input_count.py +++ b/tests/studio/test_auth_form_input_count.py @@ -51,7 +51,7 @@ def _conditional_extent(src: str) -> tuple[int, int]: def test_hasbootstrappassword_constant_is_derived_from_bootstrap_window_value(): """The guard must read from window.__UNSLOTH_BOOTSTRAP__, matching the backend's bootstrap-injection contract in studio/backend/main.py::_inject_bootstrap.""" - src = AUTH_FORM.read_text() + src = AUTH_FORM.read_text(encoding = "utf-8") assert "const hasBootstrapPassword = Boolean(window.__UNSLOTH_BOOTSTRAP__?.password);" in src, ( "hasBootstrapPassword constant missing or its derivation drifted; " "this is the gate that hides the Current password input on first boot" @@ -61,7 +61,7 @@ def test_hasbootstrappassword_constant_is_derived_from_bootstrap_window_value(): def test_exactly_one_hasBootstrapPassword_conditional_exists(): """Only one `!hasBootstrapPassword` JSX check is allowed; a second would split rendering into branches and likely hide or duplicate the New / Confirm inputs.""" - src = AUTH_FORM.read_text() + src = AUTH_FORM.read_text(encoding = "utf-8") count = src.count("!hasBootstrapPassword") assert count == 1, ( f"expected exactly one !hasBootstrapPassword usage, found {count}; " @@ -72,7 +72,7 @@ def test_exactly_one_hasBootstrapPassword_conditional_exists(): def test_current_password_input_is_inside_the_hasBootstrapPassword_conditional(): """`id="current-password"` must sit inside `{!hasBootstrapPassword && (...)}`, else it renders on first boot too, regressing the pre-#5490 UX that PR #5545 restores.""" - src = AUTH_FORM.read_text() + src = AUTH_FORM.read_text(encoding = "utf-8") s, e = _conditional_extent(src) idx = src.find('id="current-password"') assert idx != -1, "the Current password input was removed entirely" @@ -86,7 +86,7 @@ def test_current_password_input_is_inside_the_hasBootstrapPassword_conditional() def test_new_password_input_is_outside_the_hasBootstrapPassword_conditional(): """`id="new-password"` must sit outside `{!hasBootstrapPassword && (...)}`, else it disappears on admin-forced resets, regressing PR #5490.""" - src = AUTH_FORM.read_text() + src = AUTH_FORM.read_text(encoding = "utf-8") s, e = _conditional_extent(src) idx = src.find('id="new-password"') assert idx != -1, "the New password input was removed entirely" @@ -99,7 +99,7 @@ def test_new_password_input_is_outside_the_hasBootstrapPassword_conditional(): def test_confirm_password_input_is_outside_the_hasBootstrapPassword_conditional(): """Same as New password, for `id="confirm-password"`.""" - src = AUTH_FORM.read_text() + src = AUTH_FORM.read_text(encoding = "utf-8") s, e = _conditional_extent(src) idx = src.find('id="confirm-password"') assert idx != -1, "the Confirm password input was removed entirely" @@ -114,7 +114,7 @@ def test_change_password_jsx_declares_exactly_three_password_inputs(): """The change-password JSX block (`{!isLoginMode && (...)}`) must declare exactly current/new/confirm; a fourth would break the 2-input first-boot contract (the conditional only hides Current).""" - src = AUTH_FORM.read_text() + src = AUTH_FORM.read_text(encoding = "utf-8") start = src.find("{!isLoginMode && (") assert start != -1, ( "the change-password JSX subtree marker {!isLoginMode && (...)} " @@ -147,7 +147,7 @@ def test_change_password_jsx_declares_exactly_three_password_inputs(): def test_login_jsx_declares_exactly_one_password_input(): """The login JSX block (`isLoginMode && (...)`) must declare exactly one password input (the bootstrap password pasted from the CLI); a second breaks the per-mode matrix.""" - src = AUTH_FORM.read_text() + src = AUTH_FORM.read_text(encoding = "utf-8") start = src.find("{isLoginMode && (") assert start != -1, "the login JSX subtree marker is missing" depth = 1 @@ -163,18 +163,20 @@ def test_login_jsx_declares_exactly_one_password_input(): ids = re.findall(r'id="([a-z-]+)"', subtree) # Lock the count, not the spelling, so a rename does not falsely fail. pw_ids = [x for x in ids if "password" in x] - assert len(pw_ids) == 1, ( - f"login JSX must declare exactly one password-typed input; " f"found {pw_ids!r}" - ) + assert ( + len(pw_ids) == 1 + ), f"login JSX must declare exactly one password-typed input; found {pw_ids!r}" def test_auth_flow_routes_do_not_mount_global_settings(): - root = (FRONTEND / "app/routes/__root.tsx").read_text() + root = (FRONTEND / "app/routes/__root.tsx").read_text(encoding = "utf-8") assert "{!isAuthFlowRoute && }" in root assert "useSettingsDialogStore.getState().closeDialog();" in root assert "if (isAuthFlowRoute) return;" in root for route in ("login", "change-password", "onboarding"): - assert "isAuthFlow: true" in (FRONTEND / f"app/routes/{route}.tsx").read_text() + assert "isAuthFlow: true" in (FRONTEND / f"app/routes/{route}.tsx").read_text( + encoding = "utf-8" + ) def test_auth_redirect_targets_are_idempotent_and_concurrent(tmp_path: Path): @@ -190,7 +192,7 @@ def test_auth_redirect_targets_are_idempotent_and_concurrent(tmp_path: Path): pytest.skip("node --experimental-strip-types not available") source = ( - AUTH_API.read_text() + AUTH_API.read_text(encoding = "utf-8") .replace('from "@/lib/api-base"', 'from "./stubs.mjs"') .replace('from "./session"', 'from "./stubs.mjs"') ) diff --git a/tests/studio/test_cancel_atomicity.py b/tests/studio/test_cancel_atomicity.py index 142cc2247a..391ef043d7 100644 --- a/tests/studio/test_cancel_atomicity.py +++ b/tests/studio/test_cancel_atomicity.py @@ -9,7 +9,7 @@ from pathlib import Path SOURCE_PATH = Path(__file__).resolve().parents[2] / "studio" / "backend" / "routes" / "inference.py" -_SRC = SOURCE_PATH.read_text() +_SRC = SOURCE_PATH.read_text(encoding = "utf-8") _TREE = ast.parse(_SRC) diff --git a/tests/studio/test_cancel_id_wiring.py b/tests/studio/test_cancel_id_wiring.py index 651dfbf3de..fba0e814f6 100644 --- a/tests/studio/test_cancel_id_wiring.py +++ b/tests/studio/test_cancel_id_wiring.py @@ -13,10 +13,14 @@ from pathlib import Path WORKSPACE = Path(__file__).resolve().parents[2] -MODELS_SRC = (WORKSPACE / "studio/backend/models/inference.py").read_text() -ROUTES_SRC = (WORKSPACE / "studio/backend/routes/inference.py").read_text() -ADAPTER_SRC = (WORKSPACE / "studio/frontend/src/features/chat/api/chat-adapter.ts").read_text() -API_TYPES_SRC = (WORKSPACE / "studio/frontend/src/features/chat/types/api.ts").read_text() +MODELS_SRC = (WORKSPACE / "studio/backend/models/inference.py").read_text(encoding = "utf-8") +ROUTES_SRC = (WORKSPACE / "studio/backend/routes/inference.py").read_text(encoding = "utf-8") +ADAPTER_SRC = (WORKSPACE / "studio/frontend/src/features/chat/api/chat-adapter.ts").read_text( + encoding = "utf-8" +) +API_TYPES_SRC = (WORKSPACE / "studio/frontend/src/features/chat/types/api.ts").read_text( + encoding = "utf-8" +) def _find_class(tree: ast.AST, name: str) -> ast.ClassDef | None: diff --git a/tests/studio/test_chat_preset_builtin_invariants.py b/tests/studio/test_chat_preset_builtin_invariants.py index 3ca09dadda..31b2a2b733 100644 --- a/tests/studio/test_chat_preset_builtin_invariants.py +++ b/tests/studio/test_chat_preset_builtin_invariants.py @@ -40,13 +40,15 @@ def _require_node(): def _ensure_harness(): TEMP.mkdir(parents = True, exist_ok = True) (TEMP / "register.mjs").write_text( - "import { register } from 'node:module';\nregister('./loader.mjs', import.meta.url);\n" + "import { register } from 'node:module';\nregister('./loader.mjs', import.meta.url);\n", + encoding = "utf-8", ) (TEMP / "loader.mjs").write_text( "export function resolve(specifier, context, next) {\n" " if (specifier.endsWith('/types/runtime')) return next(specifier + '.ts', context);\n" " return next(specifier, context);\n" - "}\n" + "}\n", + encoding = "utf-8", ) @@ -54,7 +56,7 @@ def _run(script: str): _require_node() _ensure_harness() script_path = TEMP / "run.mts" - script_path.write_text(script) + script_path.write_text(script, encoding = "utf-8") env = dict(os.environ, NODE_NO_WARNINGS = "1") result = subprocess.run( [ diff --git a/tests/studio/test_chat_prompt_variables.py b/tests/studio/test_chat_prompt_variables.py index dcf318b5fa..6ef3b79ae9 100644 --- a/tests/studio/test_chat_prompt_variables.py +++ b/tests/studio/test_chat_prompt_variables.py @@ -6,7 +6,9 @@ from pathlib import Path WORKSPACE = Path(__file__).resolve().parents[2] -ADAPTER_SRC = (WORKSPACE / "studio/frontend/src/features/chat/api/chat-adapter.ts").read_text() +ADAPTER_SRC = (WORKSPACE / "studio/frontend/src/features/chat/api/chat-adapter.ts").read_text( + encoding = "utf-8" +) def _function_source(name: str) -> str: diff --git a/tests/studio/test_chat_response_details_ui_contract.py b/tests/studio/test_chat_response_details_ui_contract.py index 1183151b54..aa1a5cd965 100644 --- a/tests/studio/test_chat_response_details_ui_contract.py +++ b/tests/studio/test_chat_response_details_ui_contract.py @@ -20,14 +20,14 @@ CHAT_TAB_TSX = REPO / "studio/frontend/src/features/settings/tabs/chat-tab.tsx" def test_assistant_more_menu_exposes_response_details_action(): - src = THREAD_TSX.read_text() + src = THREAD_TSX.read_text(encoding = "utf-8") assert "MessageResponseDetailsSheet" in src assert "See response details" in src assert "setDetailsOpen(true)" in src def test_response_details_sheet_uses_unsloth_sheet_and_key_sections(): - src = DETAILS_TSX.read_text() + src = DETAILS_TSX.read_text(encoding = "utf-8") assert "SheetContent" in src assert "Response details" in src assert "MessageResponseModelBadge" in src @@ -45,17 +45,17 @@ def test_response_details_sheet_uses_unsloth_sheet_and_key_sections(): def test_response_model_badge_is_user_configurable_and_rendered_once_per_message(): - prefs_src = CHAT_PREFS_TS.read_text() - chat_tab_src = CHAT_TAB_TSX.read_text() - thread_src = THREAD_TSX.read_text() - reasoning_src = REASONING_TSX.read_text() + prefs_src = CHAT_PREFS_TS.read_text(encoding = "utf-8") + chat_tab_src = CHAT_TAB_TSX.read_text(encoding = "utf-8") + thread_src = THREAD_TSX.read_text(encoding = "utf-8") + reasoning_src = REASONING_TSX.read_text(encoding = "utf-8") assert "showResponseModel: boolean" in prefs_src assert "showResponseModel: false" in prefs_src assert "showResponseModel: saved?.showResponseModel ?? false" in prefs_src assert "Show response model" in chat_tab_src assert "setShowResponseModel" in chat_tab_src - details_src = DETAILS_TSX.read_text() + details_src = DETAILS_TSX.read_text(encoding = "utf-8") assert ( "aui-response-model-badge pointer-events-none relative inline-flex min-h-5" in details_src ) @@ -76,7 +76,7 @@ def test_response_model_badge_is_user_configurable_and_rendered_once_per_message def test_reasoning_keeps_streaming_height_cap_through_automatic_collapse(): - src = REASONING_TSX.read_text() + src = REASONING_TSX.read_text(encoding = "utf-8") assert "const [retainStreamingHeight, setRetainStreamingHeight]" in src assert "setRetainStreamingHeight(false)" in src @@ -91,7 +91,7 @@ def test_reasoning_clears_manual_open_on_a_new_stream(): isOpen is `(streaming && !dismissed) || manualOpen` and manualOpen is only settable while idle, so the new-stream reset has to clear it too. """ - src = REASONING_TSX.read_text() + src = REASONING_TSX.read_text(encoding = "utf-8") marker = "setDismissedWhileStreaming(false)" start = src.find(marker) @@ -101,7 +101,7 @@ def test_reasoning_clears_manual_open_on_a_new_stream(): def test_response_details_metadata_is_persisted_without_backend_schema_change(): - src = ADAPTER_TS.read_text() + src = ADAPTER_TS.read_text(encoding = "utf-8") assert "interface ResponseDetailsMetadata" in src assert "buildResponseDetails" in src assert "responseDetails: buildResponseDetails(finishedAt)" in src diff --git a/tests/studio/test_chat_title_generation.py b/tests/studio/test_chat_title_generation.py index 6a47cfbce4..3a8cbb95f9 100644 --- a/tests/studio/test_chat_title_generation.py +++ b/tests/studio/test_chat_title_generation.py @@ -41,7 +41,7 @@ def _balanced_block(src: str, anchor: str) -> str: def test_title_model_prompt_targets_conversation_topic(): block = _source_until( - RUNTIME_TSX.read_text(), + RUNTIME_TSX.read_text(encoding = "utf-8"), "async function generateTitleWithModel", "\nconst inflightTitleByKey", ) @@ -54,7 +54,7 @@ def test_title_model_prompt_targets_conversation_topic(): def test_title_model_payload_includes_optional_assistant_reply(): block = _source_until( - RUNTIME_TSX.read_text(), + RUNTIME_TSX.read_text(encoding = "utf-8"), "async function generateTitleWithModel", "\nconst inflightTitleByKey", ) @@ -71,7 +71,7 @@ def test_title_model_payload_includes_optional_assistant_reply(): def test_generate_title_passes_first_assistant_reply_after_first_user(): block = _balanced_block( - RUNTIME_TSX.read_text(), + RUNTIME_TSX.read_text(encoding = "utf-8"), "async generateTitle(remoteId", ) @@ -84,7 +84,7 @@ def test_generate_title_passes_first_assistant_reply_after_first_user(): def test_tool_call_only_first_assistant_still_uses_first_user_message(): - source = RUNTIME_TSX.read_text() + source = RUNTIME_TSX.read_text(encoding = "utf-8") extract_block = " ".join(_balanced_block(source, "function extractTextParts").split()) generate_block = " ".join(_balanced_block(source, "async generateTitle(remoteId").split()) @@ -104,7 +104,7 @@ def test_tool_call_only_first_assistant_still_uses_first_user_message(): def test_auto_title_disabled_uses_deterministic_user_text_fallback(): block = _balanced_block( - RUNTIME_TSX.read_text(), + RUNTIME_TSX.read_text(encoding = "utf-8"), "async generateTitle(remoteId", ) auto_title_off = _balanced_block(block, "if (!autoTitle)") @@ -114,7 +114,7 @@ def test_auto_title_disabled_uses_deterministic_user_text_fallback(): def test_model_failure_still_falls_back_to_user_text(): - source = RUNTIME_TSX.read_text() + source = RUNTIME_TSX.read_text(encoding = "utf-8") model_block = _source_until( source, "async function generateTitleWithModel", @@ -130,7 +130,7 @@ def test_model_failure_still_falls_back_to_user_text(): def test_title_normalizer_still_enforces_output_constraints(): block = _source_until( - RUNTIME_TSX.read_text(), + RUNTIME_TSX.read_text(encoding = "utf-8"), "async function generateTitleWithModel", "\nconst inflightTitleByKey", ) diff --git a/tests/studio/test_cli_run_alias.py b/tests/studio/test_cli_run_alias.py index 498ebbdf4d..98b5d4f76c 100644 --- a/tests/studio/test_cli_run_alias.py +++ b/tests/studio/test_cli_run_alias.py @@ -17,7 +17,7 @@ def _module_calls(source: str): def test_top_level_run_alias_registered(): """`app.command("run", ...)` must be invoked with studio_run as its target.""" - source = _CLI_INIT.read_text() + source = _CLI_INIT.read_text(encoding = "utf-8") # Find ``app.command("run", ...)`` call -- the decorator-call form. found_decorator_call = False @@ -46,7 +46,7 @@ def test_top_level_run_alias_registered(): def test_studio_run_imported_for_alias(): """The alias must wire up to the studio.run function, not redefine it.""" - source = _CLI_INIT.read_text() + source = _CLI_INIT.read_text(encoding = "utf-8") tree = ast.parse(source) has_import = False for node in ast.walk(tree): diff --git a/tests/studio/test_cli_studio_defaults.py b/tests/studio/test_cli_studio_defaults.py index a39956fab0..17ff23e66c 100644 --- a/tests/studio/test_cli_studio_defaults.py +++ b/tests/studio/test_cli_studio_defaults.py @@ -48,20 +48,19 @@ def _find_typer_option_default(source: str, func_name: str, long_option: str): def test_studio_default_host_is_loopback(): """`unsloth studio` (studio_default) --host default must be 127.0.0.1.""" - source = _STUDIO_CMD_PY.read_text() + source = _STUDIO_CMD_PY.read_text(encoding = "utf-8") host_default = _find_typer_option_default(source, "studio_default", "--host") assert ( host_default is not None ), "Could not find --host typer.Option default in studio_default()" - assert host_default == "127.0.0.1", ( - f"studio_default() --host default must be '127.0.0.1' (loopback) " - f"but got '{host_default}'." - ) + assert ( + host_default == "127.0.0.1" + ), f"studio_default() --host default must be '127.0.0.1' (loopback) but got '{host_default}'." def test_studio_run_host_is_loopback(): """`unsloth studio run` --host default must be 127.0.0.1.""" - source = _STUDIO_CMD_PY.read_text() + source = _STUDIO_CMD_PY.read_text(encoding = "utf-8") host_default = _find_typer_option_default(source, "run", "--host") assert host_default is not None, "Could not find --host typer.Option default in run()" assert host_default == "127.0.0.1", ( @@ -71,7 +70,7 @@ def test_studio_run_host_is_loopback(): def test_dns_pinning_opt_out_is_registered_safe_by_default(): - source = _STUDIO_CMD_PY.read_text() + source = _STUDIO_CMD_PY.read_text(encoding = "utf-8") for func_name in ("studio_default", "run"): default = _find_typer_option_default(source, func_name, "--disable-dns-pinning") assert default is False, f"{func_name} must keep DNS pinning enabled by default" diff --git a/tests/studio/test_composer_rtl_bidi_attribute.py b/tests/studio/test_composer_rtl_bidi_attribute.py index defbfab86c..a0afe421e9 100644 --- a/tests/studio/test_composer_rtl_bidi_attribute.py +++ b/tests/studio/test_composer_rtl_bidi_attribute.py @@ -26,22 +26,22 @@ def _block_around( def test_main_composer_has_dir_auto(): # PR #5784 turned the attribute into a JSX conditional; anchor on the inner # "Message input" literal, which survives both spellings. - block = _block_around(THREAD_TSX.read_text(), '"Message input"') + block = _block_around(THREAD_TSX.read_text(encoding = "utf-8"), '"Message input"') assert 'dir="auto"' in block, 'main composer is missing dir="auto"' def test_edit_composer_has_dir_auto(): - block = _block_around(THREAD_TSX.read_text(), "aui-edit-composer-input") + block = _block_around(THREAD_TSX.read_text(encoding = "utf-8"), "aui-edit-composer-input") assert 'dir="auto"' in block, 'edit composer is missing dir="auto"' def test_compare_composer_has_dir_auto(): - block = _block_around(SHARED_TSX.read_text(), "Send to both models") + block = _block_around(SHARED_TSX.read_text(encoding = "utf-8"), "Send to both models") assert 'dir="auto"' in block, 'compare composer is missing dir="auto"' def test_ime_workflow_step_does_not_set_studio_old_pw(): - yml = WORKFLOW_YML.read_text() + yml = WORKFLOW_YML.read_text(encoding = "utf-8") drive_idx = yml.find("Drive IME + multilingual paste regression") assert drive_idx != -1, "IME drive step not found in workflow" next_step_idx = yml.find("- name:", drive_idx + 1) @@ -53,7 +53,7 @@ def test_ime_workflow_step_does_not_set_studio_old_pw(): def test_ime_pass_password_step_does_not_export_old_pw(): - yml = WORKFLOW_YML.read_text() + yml = WORKFLOW_YML.read_text(encoding = "utf-8") pass_idx = yml.find("Pass bootstrap pw for IME / i18n test") assert pass_idx != -1, "IME password setup step not found" next_step_idx = yml.find("- name:", pass_idx + 1) @@ -65,7 +65,7 @@ def test_ime_pass_password_step_does_not_export_old_pw(): def test_ime_playwright_script_does_not_read_studio_old_pw(): - src = IME_PY.read_text() + src = IME_PY.read_text(encoding = "utf-8") code_only = re.sub(r'""".*?"""', "", src, flags = re.DOTALL) assert ( "STUDIO_OLD_PW" not in code_only @@ -76,7 +76,7 @@ def test_ime_playwright_script_does_not_read_studio_old_pw(): def test_main_composer_has_stuck_compositionend_watchdog(): """Issue #5546: WSL Chrome never emits compositionend after IME commit, so the composer needs a watchdog releasing the composing flag or Send stays disabled.""" - src = THREAD_TSX.read_text() + src = THREAD_TSX.read_text(encoding = "utf-8") assert ( "IME_STUCK_TIMEOUT_MS" in src ), "main composer is missing the stuck-compositionend watchdog (issue #5546)" @@ -87,7 +87,7 @@ def test_main_composer_has_stuck_compositionend_watchdog(): def test_compare_composer_has_stuck_compositionend_watchdog(): - src = SHARED_TSX.read_text() + src = SHARED_TSX.read_text(encoding = "utf-8") assert ( "IME_STUCK_TIMEOUT_MS" in src ), "compare composer is missing the stuck-compositionend watchdog (issue #5546)" @@ -97,7 +97,7 @@ def test_compare_composer_has_stuck_compositionend_watchdog(): def test_main_composer_keydown_repins_composing_during_ime(): """Issue #5546: the keydown IME gate must re-pin composingRef so a follow-up Enter does not submit preedit text after the watchdog clears it.""" - src = THREAD_TSX.read_text() + src = THREAD_TSX.read_text(encoding = "utf-8") assert "onKeyDown" in src, "main composer is missing onKeyDown IME gate" assert "e.nativeEvent.isComposing" in src and "keyCode === 229" in src, ( "main composer keydown gate must check both nativeEvent.isComposing " @@ -108,7 +108,7 @@ def test_main_composer_keydown_repins_composing_during_ime(): def test_compare_composer_keydown_repins_composing_during_ime(): """Compare composer onKeyDown re-pins composingRef on IME keypress so a follow-up click-Send during the watchdog window does not slip preedit text.""" - src = SHARED_TSX.read_text() + src = SHARED_TSX.read_text(encoding = "utf-8") assert "composingRef.current = true" in src, ( "compare composer keydown gate must re-pin composingRef when the " "browser still considers the IME active" @@ -142,7 +142,7 @@ def _extract_block( def test_main_composer_keydown_rearms_watchdog(): """After keydown re-pins composingRef the watchdog must re-arm, else the WSL+Chrome no-compositionend path locks Send after any IME keypress (#5546).""" - src = THREAD_TSX.read_text() + src = THREAD_TSX.read_text(encoding = "utf-8") block = _extract_block(src, "const onKeyDown = useCallback") assert "refreshStuckTimer" in block, ( "main composer keydown gate must call refreshStuckTimer after " @@ -159,12 +159,11 @@ def test_main_composer_keydown_rearms_watchdog(): def test_compare_composer_keydown_rearms_watchdog(): """Same re-arm contract for the compare-mode composer.""" - src = SHARED_TSX.read_text() + src = SHARED_TSX.read_text(encoding = "utf-8") block = _extract_block(src, "function onKeyDown", opener = "{", closer = "}") - assert "refreshStuckImeTimer" in block, ( - "compare composer keydown gate must call refreshStuckImeTimer " - "after re-pinning composingRef" - ) + assert ( + "refreshStuckImeTimer" in block + ), "compare composer keydown gate must call refreshStuckImeTimer after re-pinning composingRef" def _assert_enter_guard_before_immediate_recovery(block: str, refresh_call: str) -> None: @@ -177,10 +176,9 @@ def _assert_enter_guard_before_immediate_recovery(block: str, refresh_call: str) "composingRef; candidate-confirming Enter must not submit" ) guard_block = block[enter_idx:recovery_idx] - assert "preventDefault()" in guard_block, ( - "Enter while composingRef is stuck must prevent the same key from " - "falling through to submit" - ) + assert ( + "preventDefault()" in guard_block + ), "Enter while composingRef is stuck must prevent the same key from falling through to submit" assert ( refresh_call in guard_block ), "Enter while composingRef is stuck must keep the watchdog armed" @@ -190,12 +188,12 @@ def _assert_enter_guard_before_immediate_recovery(block: str, refresh_call: str) def test_main_composer_stuck_enter_does_not_clear_before_submit(): - src = THREAD_TSX.read_text() + src = THREAD_TSX.read_text(encoding = "utf-8") block = _extract_block(src, "const onKeyDown = useCallback") _assert_enter_guard_before_immediate_recovery(block, "refreshStuckTimer") def test_compare_composer_stuck_enter_does_not_clear_before_submit(): - src = SHARED_TSX.read_text() + src = SHARED_TSX.read_text(encoding = "utf-8") block = _extract_block(src, "function onKeyDown", opener = "{", closer = "}") _assert_enter_guard_before_immediate_recovery(block, "refreshStuckImeTimer") diff --git a/tests/studio/test_export_output_path_contract.py b/tests/studio/test_export_output_path_contract.py index 8b2f829146..390c569116 100644 --- a/tests/studio/test_export_output_path_contract.py +++ b/tests/studio/test_export_output_path_contract.py @@ -32,7 +32,7 @@ def _return_tuple_arity(fn): def test_export_methods_return_three_tuple_annotation(): - tree = ast.parse(EXPORT.read_text()) + tree = ast.parse(EXPORT.read_text(encoding = "utf-8")) for fn_name in EXPORT_FNS: fn = _find_method(tree, "ExportBackend", fn_name) assert fn is not None, f"missing ExportBackend.{fn_name}" @@ -46,7 +46,7 @@ def test_export_methods_return_three_tuple_annotation(): def test_export_methods_return_three_element_tuples(): - tree = ast.parse(EXPORT.read_text()) + tree = ast.parse(EXPORT.read_text(encoding = "utf-8")) for fn_name in EXPORT_FNS: fn = _find_method(tree, "ExportBackend", fn_name) assert fn is not None @@ -57,7 +57,7 @@ def test_export_methods_return_three_element_tuples(): def test_local_save_assigns_output_path(): - tree = ast.parse(EXPORT.read_text()) + tree = ast.parse(EXPORT.read_text(encoding = "utf-8")) for fn_name in EXPORT_FNS: fn = _find_method(tree, "ExportBackend", fn_name) assert fn is not None @@ -74,7 +74,7 @@ def test_local_save_assigns_output_path(): def test_gpu_save_method_bound_for_hub_only(): - tree = ast.parse(EXPORT.read_text()) + tree = ast.parse(EXPORT.read_text(encoding = "utf-8")) fn = _find_method(tree, "ExportBackend", "export_merged_model") assert fn is not None found_pre_save_method = False @@ -103,7 +103,7 @@ def test_gpu_save_method_bound_for_hub_only(): def test_mlx_hub_only_uses_temp_directory(): - src = EXPORT.read_text() + src = EXPORT.read_text(encoding = "utf-8") assert ( src.count("tempfile.TemporaryDirectory") >= 3 ), "expected TemporaryDirectory in merged, base, and lora hub-push paths" @@ -111,7 +111,7 @@ def test_mlx_hub_only_uses_temp_directory(): def test_is_mlx_imported_from_unsloth(): - src = EXPORT.read_text() + src = EXPORT.read_text(encoding = "utf-8") assert "from unsloth import" in src head = src.split("class ExportBackend")[0] assert "_IS_MLX" in head diff --git a/tests/studio/test_frontend_dep_removal.py b/tests/studio/test_frontend_dep_removal.py index aead44f0cd..ace5955621 100644 --- a/tests/studio/test_frontend_dep_removal.py +++ b/tests/studio/test_frontend_dep_removal.py @@ -53,8 +53,7 @@ CASES: list[Case] = [ ), Case( "C3", - "removing katex is safe: streamdown/math, mermaid, " - "rehype-katex all keep it at top level", + "removing katex is safe: streamdown/math, mermaid, rehype-katex all keep it at top level", ["katex"], "PASS", [], @@ -69,8 +68,7 @@ CASES: list[Case] = [ ), Case( "C6", - "removing @radix-ui/react-slot is safe: pulled by " - "radix-ui umbrella + @assistant-ui/react", + "removing @radix-ui/react-slot is safe: pulled by radix-ui umbrella + @assistant-ui/react", ["@radix-ui/react-slot"], "PASS", [], @@ -852,7 +850,7 @@ ADV_CASES: list[AdvCase] = [ "A12", "JSDoc @import of removed pkg should FAIL", "adv12.ts", - '/** @type {import("__adv_only_pkg_l__").Foo} */\n' "const x = null;\n", + '/** @type {import("__adv_only_pkg_l__").Foo} */\nconst x = null;\n', "__adv_only_pkg_l__", "FAIL", ["__adv_only_pkg_l__"], @@ -1047,7 +1045,7 @@ PKG_FIELD_CASES: list[PkgFieldCase] = [ def run_pkg_field_cases() -> int: - head_pkg = json.loads(HEAD_PKG.read_text()) + head_pkg = json.loads(HEAD_PKG.read_text(encoding = "utf-8")) passed = 0 for pc in PKG_FIELD_CASES: synth_head = json.loads(json.dumps(head_pkg)) @@ -1110,13 +1108,13 @@ def run_pkg_field_cases() -> int: def run_adversarial_cases() -> int: ADVERSARIAL_TMP_DIR.mkdir(parents = True, exist_ok = True) - head_pkg = json.loads(HEAD_PKG.read_text()) + head_pkg = json.loads(HEAD_PKG.read_text(encoding = "utf-8")) passed = 0 for ac in ADV_CASES: # Drop the synthetic file. fpath = ADVERSARIAL_TMP_DIR / ac.filename try: - fpath.write_text(ac.content) + fpath.write_text(ac.content, encoding = "utf-8") # Base adds the target pkg; real head lacks it, so the script # treats it as removed and scans the repo (now with our file). synth_base = json.loads(json.dumps(head_pkg)) @@ -1259,7 +1257,7 @@ ENUM_CASES: list[EnumCase] = [ def run_enum_cases() -> int: - head_pkg = json.loads(HEAD_PKG.read_text()) + head_pkg = json.loads(HEAD_PKG.read_text(encoding = "utf-8")) passed = 0 ADVERSARIAL_TMP_DIR.mkdir(parents = True, exist_ok = True) for ec in ENUM_CASES: @@ -1508,7 +1506,7 @@ def run_wrapper_cases() -> int: def main() -> int: - head_pkg = json.loads(HEAD_PKG.read_text()) + head_pkg = json.loads(HEAD_PKG.read_text(encoding = "utf-8")) print(f"Running {len(CASES)} edge cases against {SCRIPT.relative_to(REPO)}") print() results: list[tuple[Case, bool, str]] = [] diff --git a/tests/studio/test_is_mlx_dispatch_gate.py b/tests/studio/test_is_mlx_dispatch_gate.py index 0e5de1b789..f233b76e4e 100644 --- a/tests/studio/test_is_mlx_dispatch_gate.py +++ b/tests/studio/test_is_mlx_dispatch_gate.py @@ -27,7 +27,7 @@ UNSLOTH_INIT = REPO_ROOT / "unsloth" / "__init__.py" def test_is_mlx_gate_uses_three_required_predicates(): """_IS_MLX must AND Darwin+arm64+importable-mlx; dropping any breaks dispatch.""" - tree = ast.parse(UNSLOTH_INIT.read_text()) + tree = ast.parse(UNSLOTH_INIT.read_text(encoding = "utf-8")) target = None for node in ast.walk(tree): diff --git a/tests/studio/test_llama_cpp_wall_clock_cap.py b/tests/studio/test_llama_cpp_wall_clock_cap.py index b7b6917092..899eb64af7 100644 --- a/tests/studio/test_llama_cpp_wall_clock_cap.py +++ b/tests/studio/test_llama_cpp_wall_clock_cap.py @@ -14,7 +14,7 @@ SOURCE_PATH = ( / "inference" / "llama_cpp.py" ) -SRC = SOURCE_PATH.read_text() +SRC = SOURCE_PATH.read_text(encoding = "utf-8") TREE = ast.parse(SRC) diff --git a/tests/studio/test_mlx_training_worker_behaviors.py b/tests/studio/test_mlx_training_worker_behaviors.py index 78b229d6e9..adffdc501b 100644 --- a/tests/studio/test_mlx_training_worker_behaviors.py +++ b/tests/studio/test_mlx_training_worker_behaviors.py @@ -15,7 +15,7 @@ def _find_func(tree, name): def test_run_mlx_training_passes_token_to_from_pretrained(): - tree = ast.parse(WORKER.read_text()) + tree = ast.parse(WORKER.read_text(encoding = "utf-8")) fn = _find_func(tree, "_run_mlx_training") assert fn is not None found = False @@ -36,7 +36,7 @@ def test_run_mlx_training_passes_token_to_from_pretrained(): def test_wandb_init_strips_secret_keys(): - src = WORKER.read_text() + src = WORKER.read_text(encoding = "utf-8") assert "_wandb_sensitive" in src, "expected a sensitive-key set near wandb.init" assert '"hf_token"' in src and '"wandb_token"' in src assert ( @@ -45,26 +45,26 @@ def test_wandb_init_strips_secret_keys(): def test_local_dataset_loader_uses_load_dataset_path(): - src = WORKER.read_text() + src = WORKER.read_text(encoding = "utf-8") assert "_resolve_mlx_local_dataset_files" in src assert "_mlx_local_dataset_loader_for_files" in src assert "data_files = all_files" in src or "data_files=all_files" in src def test_send_aliases_status_message_to_message(): - src = WORKER.read_text() + src = WORKER.read_text(encoding = "utf-8") assert 'kwargs["message"] = sm' in src or 'kwargs["message"]=sm' in src def test_slice_uses_inclusive_end_and_handles_zero(): - src = WORKER.read_text() + src = WORKER.read_text(encoding = "utf-8") assert "min(end + 1, len(ds))" in src or "min(end+1, len(ds))" in src assert "slice_start if slice_start is not None else 0" in src assert "slice_end if slice_end is not None else len(ds) - 1" in src def test_poll_stop_returns_on_broken_pipe(): - src = WORKER.read_text() + src = WORKER.read_text(encoding = "utf-8") assert "except (EOFError, OSError)" in src lines = src.splitlines() for i, line in enumerate(lines): @@ -83,7 +83,7 @@ def test_poll_stop_returns_on_broken_pipe(): def test_unsloth_zoo_mlx_imports_have_friendly_error(): - src = WORKER.read_text() + src = WORKER.read_text(encoding = "utf-8") assert "from unsloth_zoo.mlx.loader import FastMLXModel" in src assert "from unsloth_zoo.mlx.trainer import" in src assert "raise ImportError" in src diff --git a/tests/studio/test_model_picker_contracts.py b/tests/studio/test_model_picker_contracts.py index 815f68e010..93e0d3e834 100644 --- a/tests/studio/test_model_picker_contracts.py +++ b/tests/studio/test_model_picker_contracts.py @@ -23,7 +23,7 @@ FRONTEND = WORKDIR / "studio" / "frontend" / "src" def _read(rel: str) -> str: path = FRONTEND / rel assert path.exists(), f"missing source file: {path}" - return path.read_text() + return path.read_text(encoding = "utf-8") def test_models_api_sends_token_via_header_not_query(): diff --git a/tests/studio/test_studio_gguf_export_script_pin.py b/tests/studio/test_studio_gguf_export_script_pin.py index defd0d49d4..3d643dc4c3 100644 --- a/tests/studio/test_studio_gguf_export_script_pin.py +++ b/tests/studio/test_studio_gguf_export_script_pin.py @@ -12,7 +12,7 @@ from pathlib import Path SOURCE_PATH = ( Path(__file__).resolve().parents[2] / "studio" / "backend" / "core" / "export" / "export.py" ) -SRC = SOURCE_PATH.read_text() +SRC = SOURCE_PATH.read_text(encoding = "utf-8") TREE = ast.parse(SRC) diff --git a/tests/studio/test_studio_text_descender_clipping.py b/tests/studio/test_studio_text_descender_clipping.py index 98fb3b4b13..c7196a029b 100644 --- a/tests/studio/test_studio_text_descender_clipping.py +++ b/tests/studio/test_studio_text_descender_clipping.py @@ -24,7 +24,7 @@ APP_SIDEBAR = WORKDIR / "studio" / "frontend" / "src" / "components" / "app-side def _read(path: Path) -> str: assert path.exists(), f"missing source file: {path}" - return path.read_text() + return path.read_text(encoding = "utf-8") def test_model_selector_trigger_label_uses_leading_tight(): diff --git a/tests/test_fast_generate_slow_guard.py b/tests/test_fast_generate_slow_guard.py index 6bfc561e54..b32cf3c56c 100644 --- a/tests/test_fast_generate_slow_guard.py +++ b/tests/test_fast_generate_slow_guard.py @@ -13,7 +13,7 @@ UTILS = os.path.join(HERE, "unsloth", "models", "_utils.py") def _load_factory(): - src = open(UTILS).read() + src = open(UTILS, encoding = "utf-8").read() for node in ast.parse(src).body: if isinstance(node, ast.FunctionDef) and node.name == "make_fast_generate_wrapper": ns = {"functools": functools} diff --git a/tests/test_fp8_device_context.py b/tests/test_fp8_device_context.py index 2eea35f4e6..1f72a23ec7 100644 --- a/tests/test_fp8_device_context.py +++ b/tests/test_fp8_device_context.py @@ -78,7 +78,7 @@ class _LaunchVisitor(ast.NodeVisitor): def _load_device_context_helper(fake_torch: _FakeTorch): - source = FP8_SOURCE.read_text() + source = FP8_SOURCE.read_text(encoding = "utf-8") tree = ast.parse(source) for node in tree.body: if isinstance(node, ast.FunctionDef) and node.name == "_fp8_triton_device_context": @@ -144,7 +144,7 @@ def test_fp8_device_context_is_noop_for_non_cuda_tensor() -> None: def test_fp8_triton_launches_enter_tensor_device_context() -> None: - tree = ast.parse(FP8_SOURCE.read_text()) + tree = ast.parse(FP8_SOURCE.read_text(encoding = "utf-8")) function_names = {node.name for node in ast.walk(tree) if isinstance(node, ast.FunctionDef)} assert "_fp8_triton_device_context" in function_names diff --git a/tests/test_gemma4_chat_template.py b/tests/test_gemma4_chat_template.py index cfbc81f736..fa9e253965 100644 --- a/tests/test_gemma4_chat_template.py +++ b/tests/test_gemma4_chat_template.py @@ -14,7 +14,7 @@ CHAT_TEMPLATES_PATH = os.path.join( def _extract_template(name): - src = open(CHAT_TEMPLATES_PATH).read() + src = open(CHAT_TEMPLATES_PATH, encoding = "utf-8").read() pattern = rf'{re.escape(name)}\s*=\s*\\\n"""(.*?)"""' m = re.search(pattern, src, flags = re.DOTALL) assert m, f"Could not extract {name} from chat_templates.py" diff --git a/tests/test_gemma_2b_mapper_key.py b/tests/test_gemma_2b_mapper_key.py index 31edacfce4..5435eb22f8 100644 --- a/tests/test_gemma_2b_mapper_key.py +++ b/tests/test_gemma_2b_mapper_key.py @@ -18,7 +18,7 @@ MAPPER_PATH = os.path.join(os.path.dirname(__file__), os.pardir, "unsloth", "mod def _load_mappers(): - with open(MAPPER_PATH) as f: + with open(MAPPER_PATH, encoding = "utf-8") as f: source = f.read() namespace = {} exec(compile(source, MAPPER_PATH, "exec"), namespace) diff --git a/tests/test_generate_kwarg_gate.py b/tests/test_generate_kwarg_gate.py index 6d1379d3a9..00b3ddf6ee 100644 --- a/tests/test_generate_kwarg_gate.py +++ b/tests/test_generate_kwarg_gate.py @@ -9,7 +9,7 @@ VISION = os.path.join(HERE, "unsloth", "models", "vision.py") def _load_helper(): - src = open(VISION).read() + src = open(VISION, encoding = "utf-8").read() mod = ast.parse(src) for node in mod.body: if isinstance(node, ast.FunctionDef) and node.name == "_unsloth_generate_accepts_kwarg": diff --git a/tests/test_gradient_checkpointing_restore.py b/tests/test_gradient_checkpointing_restore.py index 4f9f3faccc..ee9ef163b6 100644 --- a/tests/test_gradient_checkpointing_restore.py +++ b/tests/test_gradient_checkpointing_restore.py @@ -28,8 +28,8 @@ import re from pathlib import Path _ROOT = Path(__file__).resolve().parent.parent / "unsloth" / "models" -_RL = (_ROOT / "rl.py").read_text() -_RL_REPLACEMENTS = (_ROOT / "rl_replacements.py").read_text() +_RL = (_ROOT / "rl.py").read_text(encoding = "utf-8") +_RL_REPLACEMENTS = (_ROOT / "rl_replacements.py").read_text(encoding = "utf-8") # The single-line ternary form used at the trainer call sites: # ._unsloth_gradient_checkpointing if hasattr(, '...') else getattr(, 'gradient_checkpointing', True) @@ -162,7 +162,7 @@ def test_recording_sites_are_real_module_code(): # string. Assert it's present at the choke point (patch_peft_model, so loaded adapters # are covered) and at the pre-wrapped pass-through, both of which bypass the old # get_peft_model-only recording. - llama = (_ROOT / "llama.py").read_text() + llama = (_ROOT / "llama.py").read_text(encoding = "utf-8") tree = ast.parse(llama) def assigns_marker(node): diff --git a/tests/test_import_fixes_drift.py b/tests/test_import_fixes_drift.py index 0bee68f940..8596bf259d 100644 --- a/tests/test_import_fixes_drift.py +++ b/tests/test_import_fixes_drift.py @@ -704,7 +704,7 @@ def test_accelerate_find_device_skips_empty_logits(): def test_accelerate_patch_wired_into_gpu_init(): """The patch must be installed at startup, not only importable.""" source = Path(__file__).resolve().parent.parent / "unsloth" / "_gpu_init.py" - source = source.read_text() + source = source.read_text(encoding = "utf-8") assert "patch_accelerate_recursively_apply()" in source, ( "DRIFT DETECTED: patch_accelerate_recursively_apply is defined but " "never called in _gpu_init.py, so real imports never install it." diff --git a/tests/test_loader_glob_skip.py b/tests/test_loader_glob_skip.py index ade9e89fde..c37515a8cc 100644 --- a/tests/test_loader_glob_skip.py +++ b/tests/test_loader_glob_skip.py @@ -116,7 +116,7 @@ class TestLoaderSourceHasGuard(unittest.TestCase): loader_path = os.path.join( os.path.dirname(__file__), os.pardir, "unsloth", "models", "loader.py" ) - with open(loader_path) as f: + with open(loader_path, encoding = "utf-8") as f: source = f.read() lines = source.splitlines() diff --git a/tests/test_multi_image_grpo_chunking.py b/tests/test_multi_image_grpo_chunking.py index ea142ce1ef..350dc403cd 100644 --- a/tests/test_multi_image_grpo_chunking.py +++ b/tests/test_multi_image_grpo_chunking.py @@ -12,7 +12,7 @@ SOURCE_PATH = os.path.join(REPO_ROOT, "unsloth", "models", "rl_replacements.py") def _read_source() -> str: - with open(SOURCE_PATH, "r") as fh: + with open(SOURCE_PATH, "r", encoding = "utf-8") as fh: return fh.read() diff --git a/tests/test_offload_embedding_hooks.py b/tests/test_offload_embedding_hooks.py index b8be603b2a..4739372e15 100644 --- a/tests/test_offload_embedding_hooks.py +++ b/tests/test_offload_embedding_hooks.py @@ -11,7 +11,7 @@ VISION = os.path.join(HERE, "unsloth", "models", "vision.py") def _load_installer(): - src = open(VISION).read() + src = open(VISION, encoding = "utf-8").read() mod = ast.parse(src) for node in mod.body: if isinstance(node, ast.FunctionDef) and node.name == "_install_offload_embedding_hooks": diff --git a/tests/test_offload_tied_guard.py b/tests/test_offload_tied_guard.py index 096fba116d..f7f51d6913 100644 --- a/tests/test_offload_tied_guard.py +++ b/tests/test_offload_tied_guard.py @@ -11,7 +11,7 @@ VISION = os.path.join(HERE, "unsloth", "models", "vision.py") def _load_fn(): - src = open(VISION).read() + src = open(VISION, encoding = "utf-8").read() mod = ast.parse(src) for node in mod.body: if isinstance(node, ast.FunctionDef) and node.name == "_embeddings_are_tied": diff --git a/tests/test_source_read_encoding.py b/tests/test_source_read_encoding.py new file mode 100644 index 0000000000..07af605fe0 --- /dev/null +++ b/tests/test_source_read_encoding.py @@ -0,0 +1,1252 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Guard: tests that read checked-in files must name their encoding. + +`Path.read_text()` and `open()` with no encoding use `locale.getencoding()`: +UTF-8 on the Linux and macOS runners, cp1252 on a stock Windows install. A test +that reads a repo file that way passes in CI and raises UnicodeDecodeError for a +Windows contributor as soon as that file gains a non-ASCII byte, which the +source-scanning tests do constantly: + + studio/backend/routes/inference.py carries the DeepSeek tool-call token + regexes, so it holds U+FF5C and U+2581. Reading it as cp1252 dies on + "byte 0x81", taking test_cancel_atomicity.py and test_cancel_id_wiring.py + out at collection time. + +A call is an offence when it does un-pinned text I/O and either of two things +holds. It runs at import, where nothing can see a tmp_path fixture yet. Or the +path it reads anchors on something checked in: a module-level constant or +import, which a fixture parameter can never be, `__file__`, or a relative +literal that names a file actually present in the tree. Anchoring is what +decides the second one, followed through `/` joins, path methods, the locals +and loop variables of the enclosing function, and the parameters of helpers +every caller hands a checked-in path. So `for p in (_B / "routes").rglob("*.py")` +is in scope, `_source(LOADER_PATH)` puts the bare read inside `_source` in +scope, and anything growing out of a tmp_path stays out. That reaches test +bodies, where the same failure lands one step later: + + test_gemma4_chat_template.py opens unsloth/chat_templates.py through a + helper its tests call, and cp1252 cannot decode that file ("byte 0x90"). + test_consent_gate.py reads routes/inference.py as `(_BACKEND / rel)` and + test_gguf_load_cache_reuse.py as `Path(__file__).parent.parent / ...`, both + dying on the same 0x81 the two cancel modules hit at collection. + +Every question the rules ask is answered by the call, its path expression, or +the call sites of the helper it sits in, which keeps them mechanical enough to +enforce with no allowlist and quiet about temp-dir I/O, where the platform +default is harmless and the test wrote the bytes itself. + +Three shapes are consequently out of reach, all fixed by hand and none decidable +from the call. A path a helper hands back rather than takes in, as +`for path in _iter_caller_files()` does in test_security_gate_consistency.py, +says nothing about itself at the read. Text read from a checked-in file and +then written back to a tmp_path, at test_studio_install_workspace_guard.py:851 +and test_scan_packages.py:40, is unsafe only because of where the string came +from. And a read inside a `python -c` snippet, as test_studio_import_no_torch.py +and test_e2e_no_torch_sandbox.py build for their subprocess tests, runs in a +child interpreter this scan never parses: the snippet is an f-string whose paths +are replacement fields, so recovering it would mean evaluating the +interpolation. Reviewers have to catch those three; running the suite under +LC_ALL=C is the cheapest way to find them, since ASCII rejects every byte cp1252 +does and more. +""" + +# `str | None` below is evaluated at import on Python 3.9 without this, and +# pyproject declares requires-python = ">=3.9,<3.15". +from __future__ import annotations + +import ast +import os +import subprocess +from pathlib import Path + +TESTS = Path(__file__).resolve().parent +REPO = TESTS.parent +# Both trees ship to Windows contributors, and separate CI jobs collect them +# (repo-cpu-tests and the studio-backend matrix), so the rule covers both. +# Not a hand-written list: studio/backend/hub/tests and unsloth/kernels/moe/tests +# are already here, and the next one has to be covered the day it lands. +SKIP_DIRS = {".git", ".venv", "build", "dist", "frontend", "node_modules", "site-packages"} + + +def _walked_test_files(repo: Path): + """Every *.py under a tests directory, found by walking.""" + found = [] + for dirpath, dirnames, filenames in os.walk(repo): + dirnames[:] = sorted(d for d in dirnames if d not in SKIP_DIRS) + if "tests" not in Path(dirpath).relative_to(repo).parts: + continue + found.extend(Path(dirpath) / f for f in filenames if f.endswith(".py")) + return found + + +def _tracked_test_files(repo: Path): + """The same, but only what git is actually tracking. + + A walk picks up whatever happens to be lying in the checkout: a scratch + directory, a nested worktree, a vendored dependency. None of those are ours + to police, and a single syntax error in one would fail this test for + everybody who has one. Asking git keeps the promise the docstring makes. + """ + try: + listed = subprocess.run( + ["git", "-C", str(repo), "ls-files", "-z", "--", "*.py"], + capture_output = True, + timeout = 60, + ) + except (OSError, subprocess.SubprocessError): + return None + if listed.returncode != 0: + return None # not a checkout, so fall back to walking + names = listed.stdout.decode("utf-8", errors = "replace").split("\0") + return [ + repo / name + for name in names + if name and "tests" in Path(name).parts and not SKIP_DIRS.intersection(Path(name).parts) + ] + + +SOURCES = _tracked_test_files(REPO) +if SOURCES is None: + SOURCES = _walked_test_files(REPO) +GUARDED_METHODS = {"read_text", "write_text"} +# Openers that are somebody else's are recognised by the file's own imports +# rather than a fixed list, so `import tarfile as tf` and `from PIL import +# Image` are both covered without naming either. +# These wrap their stream in a TextIOWrapper for a "t" mode, which takes the +# platform default exactly like builtin open. Unlike open they default to "rb", +# so only an explicit text mode is in scope. lzma takes encoding keyword-only. +COMPRESSED_OPENERS = {"bz2": 3, "gzip": 3, "lzma": None} +# Wrappers that stay lazy, so draining one drains what it was given. +LAZY_ADAPTERS = {"enumerate", "filter", "islice", "map", "reversed", "zip"} +# Callables that drain a generator argument immediately. +EAGER_CONSUMERS = { + "all", + "any", + "dict", + "frozenset", + "list", + "max", + "min", + "next", + "set", + "sorted", + "sum", + "tuple", +} +# Values that re-select the platform default when passed as the encoding. +PLATFORM_DEFAULT_ENCODINGS = (None, "locale") +# `Path.read_text(p)` is the unbound spelling of `p.read_text()`: same API, same +# platform default, but the instance takes the first slot so every argument +# shifts one place right. +PATH_CLASSES = {"Path", "PosixPath", "PurePath", "WindowsPath"} +# Modules whose `open` IS the builtin: same signature, same platform default. +BUILTIN_OPEN_MODULES = {"builtins", "io"} +# Receivers `self.SOURCE` and `cls.SOURCE` reach a class attribute through. +SELF_NAMES = {"cls", "self"} +# A module-level name is normally an anchor, since a fixture cannot reach one. +# These build a directory the run owns, so a name rooted in one is temp I/O +# however it is spelled, and the platform default there is harmless. +TEMP_FACTORIES = { + "NamedTemporaryFile", + "TemporaryDirectory", + "gettempdir", + "mkdtemp", + "mkstemp", +} +# Functions that hand back a path still pointing at their first argument. An +# unlisted call is left unresolved: a helper may well return a temp copy of what +# it was given, and following it would put test-created files back in scope. +PATH_FUNCTIONS = { + "abspath", + "dirname", + "expanduser", + "fspath", + "join", + "normpath", + "realpath", + "relpath", + "str", +} +# Path methods that hand back another path, so the receiver is still the anchor. +PATH_METHODS = { + "absolute", + "as_posix", + "expanduser", + "glob", + "iterdir", + "joinpath", + "resolve", + "rglob", + "with_name", + "with_stem", + "with_suffix", +} +# Where each API takes its encoding positionally, for the bound call. +ENCODING_POSITION = {"read_text": 0, "write_text": 1, "Path.open": 2, "open": 3} +# Distinct from None so that "no mode argument at all" still means text. +UNKNOWN_MODE = object() +# Stand-in for a file whose imports are not to hand, so every helper can be +# called on its own without pretending it knows what was imported. +NO_MODULES: dict = {} + + +def _static_truth(node: ast.AST): + """Whether a condition is a literal true or false, else None for "depends".""" + return bool(node.value) if isinstance(node, ast.Constant) else None + + +def _live_branches(node: ast.AST): + """The children of a branch that can actually run, or None if it is not one. + + `if False:` and the right of `False and ...` never execute, so reporting a + read there is a CI failure with no reachable cause and no correct edit. + """ + if isinstance(node, ast.If): + taken = _static_truth(node.test) + if taken is None: + return None + return [node.test, *(node.body if taken else node.orelse)] + if isinstance(node, ast.IfExp): + taken = _static_truth(node.test) + if taken is None: + return None + return [node.test, node.body if taken else node.orelse] + if isinstance(node, ast.BoolOp) and node.values: + # `and` stops at the first false operand, `or` at the first true one. + stops = isinstance(node.op, ast.Or) + live = [] + for value in node.values: + live.append(value) + if _static_truth(value) is stops: + break + return live if len(live) < len(node.values) else None + return None + + +def _callee_name(func: ast.AST): + """The bare name a callee ends in, whether or not it is qualified.""" + return func.id if isinstance(func, ast.Name) else getattr(func, "attr", None) + + +def _is_main_guard(node: ast.AST) -> bool: + """True for `if __name__ == "__main__":`, whose body never runs at import. + + The operator has to be `==`: `if __name__ != "__main__":` runs its body at + import, so treating it as script-only would invert the rule. + """ + if not isinstance(node, ast.If) or not isinstance(node.test, ast.Compare): + return False + if not all(isinstance(op, ast.Eq) for op in node.test.ops): + return False + operands = [node.test.left, *node.test.comparators] + # Either spelling: `__name__ == "__main__"` or `"__main__" == __name__`. + has_name = any(isinstance(o, ast.Name) and o.id == "__name__" for o in operands) + has_main = any(isinstance(o, ast.Constant) and o.value == "__main__" for o in operands) + return has_name and has_main + + +def _is_eager_consumer(func: ast.expr) -> bool: + """True for a callee that drains a generator argument on the spot. + + iter/zip/map/filter/enumerate/reversed hand back another lazy object, so a + genexp passed to those still has not run. + """ + if isinstance(func, ast.Attribute): + return func.attr in {"join", "extend", "update", "writelines"} + return isinstance(func, ast.Name) and func.id in EAGER_CONSUMERS + + +def _import_time_calls(tree: ast.Module): + """Yield Call nodes that run at import time. + + That is module scope, class bodies, and the bodies of module-level helpers + invoked from either. A helper is the same hazard as an inline read: + `CODE = _extract_mixed_precision_code()` runs its `read_text()` during + collection, so skipping every def would let the Windows failure back in. + + A def's body waits for a call, but its decorators and argument defaults run + when the def executes, so those are followed. Lambda bodies are skipped for + the same reason, as is everything but the outermost iterable of a generator + expression. List, set and dict comprehensions are walked in full: unlike a + genexp they run their element, filters and nested iterators immediately. + + A body is only ever entered through an executed statement, never by walking + into a def, so the "this definitely runs" property that makes the rule + allowlist-free holds. Not followed: the body of + `if __name__ == "__main__":`, which pytest never runs (its `else` arm does, + so that is walked), and non-name calls, which are left unresolved rather + than guessed at. + """ + # Defs reachable from a scope that executes at import: module body, any + # class body, and (added when the helper is entered) any def nested inside + # a helper we follow. `class F: def _load(): ...; DATA = _load()` runs + # _load while the class is constructed. + helpers: dict = {} + + def _collect(body): + scopes = [body] + while scopes: + for node in scopes.pop(): + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + helpers.setdefault(node.name, node) + elif isinstance(node, ast.ClassDef): + scopes.append(node.body) + + _collect(tree.body) + consumed = _eagerly_consumed(tree) + entered = set() + frontier = [list(tree.body)] + while frontier: + stack = frontier.pop() + while stack: + node = stack.pop() + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + # The body waits for a call; these two run right now. + stack.extend(node.decorator_list) + stack.extend(d for d in node.args.defaults if d is not None) + stack.extend(d for d in node.args.kw_defaults if d is not None) + continue + if isinstance(node, ast.Lambda): + stack.extend(d for d in node.args.defaults if d is not None) + stack.extend(d for d in node.args.kw_defaults if d is not None) + continue + if isinstance(node, ast.GeneratorExp) and id(node) not in consumed: + # Lazy: only the outermost iterable is evaluated where written. + if node.generators: + stack.append(node.generators[0].iter) + continue + if _is_main_guard(node): + stack.extend(node.orelse) # the else arm runs at import + continue + live = _live_branches(node) + if live is not None: + stack.extend(live) # the dead arm never runs, so nothing in it does + continue + if isinstance(node, ast.Call): + yield node + func = node.func + if isinstance(func, ast.Name) and func.id in helpers and func.id not in entered: + helper = helpers[func.id] + # `READS = _load(paths)` on a generator function only builds + # the generator, so its body waits for a consumer just as a + # genexp does. + if not _is_generator(helper) or id(node) in consumed: + entered.add(func.id) + body = list(helper.body) + _collect(body) # a def nested here is now callable + frontier.append(body) + stack.extend(ast.iter_child_nodes(node)) + + +def _eagerly_consumed(tree: ast.Module) -> set: + """Nodes whose lazy value is drained right where it is written. + + Covers both things that defer: a generator expression, and a call to a + generator function. Neither runs its body until something pulls from it, so + an unconsumed one has not happened yet. + """ + # `texts = (p.read_text() for p in ...)` then `list(texts)` consumes the + # generator through a name, so the name has to lead back to it. + named: dict = {} + for node in ast.walk(tree): + if isinstance(node, ast.Assign) and len(node.targets) == 1: + target = node.targets[0] + if isinstance(target, ast.Name) and isinstance(node.value, ast.GeneratorExp): + named.setdefault(target.id, node.value) + + def _resolve(node): + if isinstance(node, ast.Name) and node.id in named: + return named[node.id] + return node + + consumed = set() + for node in ast.walk(tree): + if isinstance(node, ast.Call) and _is_eager_consumer(node.func): + consumed.update(id(_resolve(a)) for a in node.args) + consumed.update(id(_resolve(k.value)) for k in node.keywords) + elif isinstance(node, (ast.For, ast.AsyncFor, ast.comprehension)): + consumed.add(id(_resolve(node.iter))) # the loop pulls every item + # `list(enumerate(_paths()))` drains _paths() as well, one wrapper down. + by_id = {id(n): n for n in ast.walk(tree)} + queue = [by_id[i] for i in list(consumed) if i in by_id] + while queue: + node = queue.pop() + if isinstance(node, ast.Call) and _callee_name(node.func) in LAZY_ADAPTERS: + for arg in node.args: + target = _resolve(arg) + if id(target) not in consumed: + consumed.add(id(target)) + queue.append(target) + return consumed + + +def _temp_rooted_names(tree: ast.Module) -> set: + """Module-level names anchored on a directory the run itself created.""" + names = set() + for node in tree.body: + value = node.value if isinstance(node, (ast.Assign, ast.AnnAssign)) else None + if value is None: + continue + if any( + isinstance(n, ast.Call) and _callee_name(n.func) in TEMP_FACTORIES + for n in ast.walk(value) + ): + targets = node.targets if isinstance(node, ast.Assign) else [node.target] + names.update(t.id for t in targets if isinstance(t, ast.Name)) + return names + + +def _non_path_names(tree: ast.Module) -> set: + """Module-level names bound to a call that plainly does not make a path. + + `response = requests.get(...)` then `response.read_text()` at import is not + pathlib I/O, and demanding an encoding there leaves no compliant edit. + """ + names = set() + for node in tree.body: + if not isinstance(node, ast.Assign) or not isinstance(node.value, ast.Call): + continue + func = node.value.func + if _is_path_preserving(func) or _callee_name(func) in PATH_METHODS: + continue + names.update(t.id for t in node.targets if isinstance(t, ast.Name)) + return names + + +def _is_generator(func) -> bool: + """True when calling this only builds a generator, leaving the body unrun. + + Yields inside a nested def belong to that def, so those do not count. + """ + stack = list(func.body) + while stack: + node = stack.pop() + if isinstance(node, (ast.Yield, ast.YieldFrom)): + return True + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)): + continue + stack.extend(ast.iter_child_nodes(node)) + return False + + +def _module_level_names(tree: ast.Module) -> set: + """Names assigned at module scope.""" + + def _bound(target): + # `SOURCE, CONFIG = Path(...), Path(...)` binds both. + if isinstance(target, ast.Name): + yield target.id + elif isinstance(target, (ast.Tuple, ast.List)): + for element in target.elts: + yield from _bound(element) + elif isinstance(target, ast.Starred): + yield from _bound(target.value) + + def _is_temp(value) -> bool: + return value is not None and any( + isinstance(n, ast.Call) and _callee_name(n.func) in TEMP_FACTORIES + for n in ast.walk(value) + ) + + names = set() + for node in tree.body: + if isinstance(node, ast.Assign): + if _is_temp(node.value): + continue # TMP = Path(tempfile.mkdtemp()) is not checked in + for target in node.targets: + names.update(_bound(target)) + elif isinstance(node, ast.AnnAssign): + if _is_temp(node.value): + continue + names.update(_bound(node.target)) + elif isinstance(node, (ast.Import, ast.ImportFrom)): + # `start._CODEX_FALLBACK_PROMPT` is a path another module defines at + # its own module scope, so the import is an anchor like any constant. + names.update((a.asname or a.name).split(".")[0] for a in node.names) + return names + + +def _local_names(func) -> set: + """Every name the function binds, so a module constant it shadows is skipped. + + Walking nested defs too over-approximates, which only ever drops a call from + the scan. + """ + args = func.args + names = {a.arg for a in [*args.posonlyargs, *args.args, *args.kwonlyargs]} + for extra in (args.vararg, args.kwarg): + if extra is not None: + names.add(extra.arg) + stack = list(ast.iter_child_nodes(func)) + while stack: + node = stack.pop() + if isinstance(node, (ast.ListComp, ast.SetComp, ast.DictComp, ast.GeneratorExp)): + # A comprehension target binds in its own scope, so it shadows + # nothing out here; the rest of the comprehension still does. + for gen in node.generators: + stack.append(gen.iter) + stack.extend(gen.ifs) + stack.extend([node.key, node.value] if isinstance(node, ast.DictComp) else [node.elt]) + continue + if isinstance(node, ast.Name) and isinstance(node.ctx, (ast.Store, ast.Del)): + names.add(node.id) + elif isinstance(node, (ast.Import, ast.ImportFrom)): + names.update((a.asname or a.name).split(".")[0] for a in node.names) + stack.extend(ast.iter_child_nodes(node)) + return names + + +def _imported_names(node) -> dict: + """Names this scope's own imports bind, mapped to where they came from. + + The name alone is not enough in either direction. `import gzip as gz` binds + a name nobody would recognise to an opener that does take an encoding, and + `from PIL.Image import open` binds a name everybody recognises to one that + does not. Keeping the origin settles both. + + Nested function bodies are left out: an import inside one is that + function's business, and treating it as the module's would let a single + local `from PIL.Image import open` turn off the builtin check everywhere. + """ + bound = {} + stack = list(ast.iter_child_nodes(node)) + while stack: + item = stack.pop() + if isinstance(item, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)): + continue + if isinstance(item, (ast.Import, ast.ImportFrom)): + bound.update(_import_bindings(item)) + else: + stack.extend(ast.iter_child_nodes(item)) + return bound + + +def _import_bindings(node) -> dict: + """What one import statement binds, mapped to where each name came from.""" + if isinstance(node, ast.Import): + return {(a.asname or a.name).split(".")[0]: a.name for a in node.names} + return { + a.asname or a.name: (f"{node.module}.{a.name}" if node.module else a.name) + for a in node.names + } + + +def _imports_at_each_call(tree: ast.Module) -> dict: + """The imports visible at every call, keyed by node id. + + A function's own imports are added on the way in and go out of view again + on the way out, which is what keeps a local alias local. Within a scope they + accumulate in statement order, so `DATA = open(p)` above a later + `from gzip import open` still resolves to the builtin it actually called. + """ + visible_at = {} + + def walk(node, visible): + if isinstance(node, ast.Call): + visible_at[id(node)] = dict(visible) + if isinstance(node, (ast.Import, ast.ImportFrom)): + visible.update(_import_bindings(node)) + return + if isinstance(node, ast.If): + # Only a branch that certainly runs may bind a name for the code + # after it; the others are explored with a copy that is thrown away. + taken = _static_truth(node.test) + walk(node.test, visible) + for arm, runs in ((node.body, taken is not False), (node.orelse, taken is not True)): + if not runs: + continue + inner = visible if taken is not None else dict(visible) + for child in arm: + walk(child, inner) + return + for child in ast.iter_child_nodes(node): + if isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)): + walk(child, dict(visible)) # its own scope, so its own copy + else: + walk(child, visible) + + walk(tree, {}) + return visible_at + + +def _open_alias(name, modules): + """What a bare callable resolves to: "builtin", a COMPRESSED_OPENERS key, or None. + + `from io import open as io_open` is the builtin under another name and + `from gzip import open as gzopen` is gzip's, while `from PIL.Image import + open` is neither and takes no encoding at all. + """ + origin = modules.get(name) + if origin is None: + return "builtin" if name == "open" else None + parts = origin.split(".") + if parts[-1] != "open": + return None + if parts[0] in BUILTIN_OPEN_MODULES or origin == "open": + return "builtin" + return parts[0] if parts[0] in COMPRESSED_OPENERS else None + + +def _origin_root(name, modules) -> str: + """The top-level module a bound name came from, or the name itself.""" + return modules.get(name, name).split(".")[0] + + +def _compressed_key(name, modules): + """The COMPRESSED_OPENERS entry this receiver resolves to, if any.""" + for candidate in (name, _origin_root(name, modules)): + if candidate in COMPRESSED_OPENERS: + return candidate + return None + + +def _is_path_class(name, modules) -> bool: + """True for a pathlib class, including under an alias. + + `from pathlib import Path as P` still puts the instance in slot 0 of an + unbound `P.read_text(SOURCE)`, so matching the bare name is not enough. + """ + if name is None: + return False + return (modules.get(name) or name).split(".")[-1] in PATH_CLASSES + + +def _is_path_attr(node: ast.AST) -> bool: + """True for a qualified path class, as in `pathlib.Path` or `pl.Path`.""" + return isinstance(node, ast.Attribute) and node.attr in PATH_CLASSES + + +def _is_path_preserving(func) -> bool: + """True for a call whose result still points at its first argument. + + Qualified spellings count: `pathlib.Path(p)` and `os.path.join(p, x)` are + the same constructors as the bare names. + """ + name = _callee_name(func) + return name in PATH_CLASSES or name in PATH_FUNCTIONS + + +def _is_module_receiver(name, modules) -> bool: + """True for a receiver that is not itself a path.""" + return ( + name in modules + or _is_path_class(name, modules) + or _compressed_key(name, modules) is not None + or _origin_root(name, modules) in BUILTIN_OPEN_MODULES + ) + + +def _path_expr(call: ast.Call, modules = NO_MODULES): + """The expression naming the file the call reads. + + Usually the receiver, but a module or the Path class in that slot means the + path is the first argument instead: `Path.read_text(REPO / "x.py")` and + `gzip.open(path, "rt")` both read their argument, not `Path` or `gzip`. + """ + func = call.func + if isinstance(func, ast.Attribute): + if _is_path_attr(func.value) or ( + isinstance(func.value, ast.Name) and _is_module_receiver(func.value.id, modules) + ): + return call.args[0] if call.args else _path_keyword(call) + return func.value + if isinstance(func, ast.Name) and _open_alias(func.id, modules) is not None: + return call.args[0] if call.args else _path_keyword(call) + return None + + +def _path_keyword(call: ast.Call): + """The path passed by keyword: `file` for open, `filename` for gzip and kin.""" + for kw in call.keywords: + if kw.arg in ("file", "filename"): + return kw.value + return None + + +def _path_root(node: ast.AST) -> ast.AST: + """Follow a path expression back to whatever it is anchored on. + + `(_BACKEND / rel).read_text()` anchors on _BACKEND and + `Path(__file__).parent / "routes"` on __file__, so joining a relative name + onto a checked-in root stays in scope. Anchoring is what decides it, not the + names further down: `tmp_path / SUBDIR` anchors on the fixture, so a + constant used as a leaf cannot drag temp-dir I/O in. + """ + while True: + if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Div): + node = node.left + elif isinstance(node, (ast.Attribute, ast.Subscript)): + if ( + isinstance(node, ast.Attribute) + and isinstance(node.value, ast.Name) + and node.value.id in SELF_NAMES + ): + return node # self.SOURCE names the class attribute, not self + node = node.value + elif isinstance(node, ast.Call): + func = node.func + # `p.rglob("*.py")` anchors on p, not on the pattern, while + # Path(x), str(x) and os.path.join(x, ...) anchor on the argument. + if isinstance(func, ast.Attribute) and func.attr in PATH_METHODS: + node = func.value + elif _is_path_preserving(func) and node.args: + node = node.args[0] + else: + # An unrecognised call says nothing about where its result + # points, so tempfile.mkdtemp() and a helper that copies its + # argument into a temp dir both stop here. + return node + else: + return node + + +def _is_checked_in_root( + node: ast.AST, + module_names: set, + shadowed, + derived = (), + attrs = (), +) -> bool: + """True when a path expression anchors on something that ships in the repo.""" + if isinstance(node, (ast.Tuple, ast.List, ast.Set)): + # `for path in (MODEL_SELECTOR, APP_SIDEBAR)` is checked in when every + # element is, which is what makes the loop variable one too. + return bool(node.elts) and all( + _is_checked_in_root( + e.value if isinstance(e, ast.Starred) else e, + module_names, + shadowed, + derived, + attrs, + ) + for e in node.elts + ) + root = _path_root(node) + if isinstance(root, ast.Constant) and isinstance(root.value, str): + # A relative literal naming something that exists here is checked in; a + # path the test creates at runtime is not in the tree to be found. + value = root.value + if not value or "\n" in value or "\0" in value or os.path.isabs(value): + return False + try: + return (REPO / value).exists() + except OSError: + return False # too long to be a name, so not one + if isinstance(root, ast.Attribute): + # `self.SOURCE`, where the class body bound SOURCE to a checked-in path. + return root.attr in attrs + if not isinstance(root, ast.Name): + return False + if root.id in derived: + return True + return root.id == "__file__" or (root.id in module_names and root.id not in shadowed) + + +def _class_path_attrs(tree: ast.Module, module_names: set) -> set: + """Class-body names bound to a checked-in path, read back as `self.NAME`. + + `class T: _SETUP_SH = ROOT / "setup.sh"` then `self._SETUP_SH.read_text()` + is as statically provable as the module-level spelling, and the repository + reads seven real source files exactly that way. + """ + attrs, mixed = set(), set() + for node in ast.walk(tree): + if not isinstance(node, ast.ClassDef): + continue + for stmt in node.body: + if isinstance(stmt, ast.Assign): + targets = stmt.targets + elif isinstance(stmt, ast.AnnAssign) and stmt.value is not None: + targets = [stmt.target] + else: + continue + bound = {t.id for t in targets if isinstance(t, ast.Name)} + # One attribute name, two classes, two meanings: only one of them is + # provable, so neither is claimed. Same rule as the local walk. + found = attrs if _is_checked_in_root(stmt.value, module_names, ()) else mixed + found.update(bound) + return attrs - mixed + + +def _reads_itself(name: str, value: ast.AST) -> bool: + """`source = source.read_text()` reads the path before replacing it. + + The name holds a checked-in path right up to that call, so the assignment + is not evidence against it; it is the very read we are looking for. + """ + if not isinstance(value, ast.Call): + return False + expr = _path_expr(value) + return isinstance(expr, ast.Name) and expr.id == name + + +def _unpack(target, value, paired: bool): + """Yield (name node, the value it is bound to) for one binding. + + A destructured target contributes every name inside it. Where the two sides + line up, as in `A, B = P1, P2`, each name takes its own element; where they + do not, as in `for name, path in CASES`, they all take the iterable, which + is the thing whose provenance is known. + """ + if isinstance(target, ast.Name): + yield target, value + return + if not isinstance(target, (ast.Tuple, ast.List)): + return + elements = None + if paired and isinstance(value, (ast.Tuple, ast.List)) and len(value.elts) == len(target.elts): + elements = value.elts + for index, element in enumerate(target.elts): + if isinstance(element, ast.Starred): + element = element.value + yield from _unpack(element, elements[index] if elements else value, paired) + + +def _checked_in_locals( + func, + module_names: set, + shadowed, + seed = (), +) -> set: + """Locals that only ever hold a checked-in path. + + `route = Path(_BACKEND_DIR) / "routes" / "inference.py"` followed by + `route.read_text()` is the same read one line apart. A name bound any other + way, or assigned anything else anywhere in the scope, is not tracked, and + the pass repeats so that a path built up over several locals still counts. + """ + assignments = [] + targets = set() + bad = set() + for node in ast.walk(func): + paired = False + if isinstance(node, ast.Assign) and len(node.targets) == 1: + target, value = node.targets[0], node.value + paired = True # `A, B = P1, P2` lines its sides up element by element + elif isinstance(node, (ast.For, ast.AsyncFor, ast.comprehension)): + # `for p in SRC_DIR.rglob("*.py")` binds p to a checked-in path too, + # and `for name, path in CASES` binds both to the same iterable. + target, value = node.target, node.iter + else: + continue + for name_node, bound in _unpack(target, value, paired): + targets.add(id(name_node)) + if not _reads_itself(name_node.id, bound): + assignments.append((name_node.id, bound)) + for node in ast.walk(func): + # A with-as or an augassign says nothing about the value it binds. + if isinstance(node, ast.Name) and isinstance(node.ctx, (ast.Store, ast.Del)): + if id(node) not in targets: + bad.add(node.id) + args = func.args + bad.update(a.arg for a in [*args.posonlyargs, *args.args, *args.kwonlyargs]) + # A parameter every caller hands a checked-in path is the exception. + bad -= set(seed) + good: set = set(seed) + while True: + grown = set(good) | { + name + for name, value in assignments + if name not in bad and _is_checked_in_root(value, module_names, shadowed, good) + } + # A name assigned a checked-in path somewhere and something else + # elsewhere stays out, since only one of the two is provable. + grown -= { + name + for name, value in assignments + if name in grown and not _is_checked_in_root(value, module_names, shadowed, good) + } + if grown == good: + return good + good = grown + + +def _unwrap_param(node: ast.AST) -> ast.AST: + """`pytest.param(SOURCE, id = "x")` is a wrapper around the real value.""" + if ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) + and node.func.attr == "param" + and node.args + ): + return node.args[0] + return node + + +def _parametrized_values(func) -> dict: + """Parameter values supplied by @pytest.mark.parametrize. + + pytest calls a parametrized test itself, so the decorator is the only call + site there is; without reading it every such parameter looks unprovable. + """ + supplied: dict = {} + for decorator in func.decorator_list: + if not isinstance(decorator, ast.Call) or len(decorator.args) < 2: + continue + if not isinstance(decorator.func, ast.Attribute) or decorator.func.attr != "parametrize": + continue + names, values = decorator.args[0], decorator.args[1] + if not isinstance(names, ast.Constant) or not isinstance(names.value, str): + continue + if not isinstance(values, (ast.List, ast.Tuple, ast.Set)): + continue + argnames = [n.strip() for n in names.value.split(",") if n.strip()] + for element in values.elts: + paired = len(argnames) > 1 and isinstance(element, (ast.Tuple, ast.List)) + row = element.elts if paired else [element] + for argname, value in zip(argnames, row): + supplied.setdefault(argname, []).append(_unwrap_param(value)) + return supplied + + +def _checked_in_params(tree: ast.Module, module_names: set) -> set: + """(function, parameter) pairs that only ever receive a checked-in path. + + `_source(LOADER_PATH)` is what tells us that the `path` parameter of + `_source` is reading a file that ships in the repo; the bare + `path.read_text()` inside it cannot say so on its own. One hop only, and a + parameter any call leaves out, or passes anything else, is not tracked. + + Definitions are held by identity, not by name. Two tests that each nest a + `_read` helper are two different functions, and merging them would let the + one handed a tmp_path rule out what the other proves. + """ + # Every definition, plus which scope it was written in, so a call resolves + # to the nearest enclosing `def` of that name the way Python resolves it. + scope_of: dict = {} + defs_in: dict = {} + + def _index(node, scope): + for child in ast.iter_child_nodes(node): + if isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef)): + defs_in.setdefault(id(scope), {}).setdefault(child.name, child) + scope_of[id(child)] = scope + _index(child, child) + elif isinstance(child, ast.ClassDef): + _index(child, scope) # a class body is not a name lookup scope + else: + _index(child, scope) + + _index(tree, tree) + + def _lookup(name, scope): + while scope is not None: + found = defs_in.get(id(scope), {}).get(name) + if found is not None: + return found + scope = scope_of.get(id(scope)) + return None + + # Which function each call sits in, so a parameter already known to hold a + # checked-in path can be passed on to the next helper. + owner: dict = {} + + def _mark(node, owning): + if isinstance(node, ast.Call): + owner[id(node)] = owning + for child in ast.iter_child_nodes(node): + nested = isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef)) + _mark(child, child if nested else owning) + + _mark(tree, None) + # Which class body each call sits in, so `self._read(...)` resolves to that + # class's method and not a same-named one in a sibling class. + in_class: dict = {} + + def _mark_class(node, cls): + if isinstance(node, ast.Call): + in_class[id(node)] = cls + for child in ast.iter_child_nodes(node): + _mark_class(child, child if isinstance(child, ast.ClassDef) else cls) + + _mark_class(tree, None) + + def _method(cls, name): + if cls is None: + return None + for stmt in cls.body: + if isinstance(stmt, (ast.FunctionDef, ast.AsyncFunctionDef)) and stmt.name == name: + return stmt + return None + + good: set = set() + while True: + grown, bad = set(), set() + for fnode in [d for scope in defs_in.values() for d in scope.values()]: + for argname, values in _parametrized_values(fnode).items(): + ok = all(_is_checked_in_root(v, module_names, ()) for v in values) + (grown if ok else bad).add((id(fnode), argname)) + # What the calling function itself can prove, recomputed each pass so a + # parameter resolved last time can feed a local this time. + scope: dict = {} + for call in ast.walk(tree): + if not isinstance(call, ast.Call): + continue + caller = owner.get(id(call)) + callee, bound = call.func, False + if isinstance(callee, ast.Name): + func = _lookup(callee.id, caller if caller is not None else tree) + elif ( + isinstance(callee, ast.Attribute) + and isinstance(callee.value, ast.Name) + and callee.value.id in SELF_NAMES + ): + # `self._read(ROOT / "x.py")` seeds `_read`'s path parameter too. + func, bound = _method(in_class.get(id(call)), callee.attr), True + else: + continue + if func is None or any(isinstance(a, ast.Starred) for a in call.args): + continue + if caller is None: + here = set() + elif id(caller) in scope: + here = scope[id(caller)] + else: + params = {p for f, p in good if f == id(caller)} + here = _checked_in_locals(caller, module_names, _local_names(caller), params) + scope[id(caller)] = here + positional = [a.arg for a in [*func.args.posonlyargs, *func.args.args]] + if bound: + positional = positional[1:] # the receiver already fills `self` + # A keyword-only parameter never takes a positional slot, so it is + # matched by name alone. + params = positional + [a.arg for a in func.args.kwonlyargs] + supplied = dict(zip(positional, call.args)) + supplied.update({k.arg: k.value for k in call.keywords if k.arg in params}) + for param in params: + value = supplied.get(param) + ok = value is not None and _is_checked_in_root(value, module_names, (), here) + (grown if ok else bad).add((id(func), param)) + grown -= bad + if grown == good: + return good + good = grown + + +def _checked_in_path_calls( + tree: ast.Module, + modules = NO_MODULES, + visible_at = None, +): + """Yield calls, at any depth, whose path is provably a checked-in file. + + The import-time walk alone leaves test bodies unguarded, and a bare read + there is the same Windows failure one step later: `_extract_template()` in + test_gemma4_chat_template.py opens unsloth/chat_templates.py, which cp1252 + cannot decode ("byte 0x90"), so the test errors rather than the collection. + + Two spellings qualify. A tmp_path arrives as a fixture parameter and a + tempfile is built in the body, so neither can be bound at module scope nor + derived from `__file__`. That keeps temp-dir I/O out of scope without an + allowlist, since there the platform default is harmless and the test wrote + the bytes itself. + """ + module_names = _module_level_names(tree) + consumed = _eagerly_consumed(tree) + visible_at = _imports_at_each_call(tree) if visible_at is None else visible_at + params = _checked_in_params(tree, module_names) + attrs = _class_path_attrs(tree, module_names) + + def visit( + node, + shadowed, + derived = frozenset(), + ): + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)): + shadowed = shadowed | _local_names(node) + # Seed with the parameters first: `p = root / "x.py"` is only + # derivable once `root` is known to hold a checked-in path. + seeded = {p for f, p in params if f == id(node)} + derived = _checked_in_locals(node, module_names, shadowed, seeded) + elif _is_main_guard(node): + # Never runs under pytest, so rule 1 skips it for the same reason. + for child in node.orelse: + yield from visit(child, shadowed, derived) + return + elif isinstance(node, ast.GeneratorExp) and id(node) not in consumed: + if node.generators: + yield from visit(node.generators[0].iter, shadowed, derived) + return + elif (live := _live_branches(node)) is not None: + for child in live: + yield from visit(child, shadowed, derived) + return + elif isinstance(node, ast.Call): + expr = _path_expr(node, visible_at.get(id(node), modules)) + if expr is not None and _is_checked_in_root( + expr, module_names, shadowed, derived, attrs + ): + yield node + for child in ast.iter_child_nodes(node): + yield from visit(child, shadowed, derived) + + yield from visit(tree, frozenset()) + + +def _open_mode(call: ast.Call, mode_index: int): + """The literal mode of an open() call, or UNKNOWN_MODE. + + A splat or a non-literal hides the mode. Defaulting those to "r" would + demand an encoding on a call that may resolve to "rb", where passing one is + a ValueError, so the contributor would have no compliant edit. + """ + if any(isinstance(a, ast.Starred) for a in call.args): + return UNKNOWN_MODE + if any(kw.arg is None for kw in call.keywords): + return UNKNOWN_MODE + if len(call.args) > mode_index: + node = call.args[mode_index] + return node.value if isinstance(node, ast.Constant) else UNKNOWN_MODE + for kw in call.keywords: + if kw.arg == "mode": + return kw.value.value if isinstance(kw.value, ast.Constant) else UNKNOWN_MODE + return "r" + + +def _is_text(call: ast.Call, mode_index: int) -> bool: + mode = _open_mode(call, mode_index) + return mode is not UNKNOWN_MODE and "b" not in str(mode) + + +def _names_encoding(call: ast.Call) -> bool: + """True only for an encoding that actually pins one. + + `encoding = None` and `encoding = "locale"` both re-select the platform + default, so the keyword being present is not enough. A `**kwargs` may carry + one we cannot see, so it counts as named rather than risking a false alarm. + """ + for kw in call.keywords: + if kw.arg is None: + return True + if kw.arg != "encoding": + continue + if isinstance(kw.value, ast.Constant) and kw.value.value in PLATFORM_DEFAULT_ENCODINGS: + return False + return True + return False + + +def _pins_encoding(call: ast.Call, position: int | None) -> bool: + """True when the call names an encoding, positionally or by keyword. + + `position` is None where the API takes it keyword-only. A splat makes the + positions meaningless, so it counts as named rather than demanding an edit + the contributor cannot make correctly. + """ + if any(isinstance(a, ast.Starred) for a in call.args): + return True + if position is not None and len(call.args) > position: + node = call.args[position] + if isinstance(node, ast.Constant): + return node.value not in PLATFORM_DEFAULT_ENCODINGS + return True + return _names_encoding(call) + + +def _offender(call: ast.Call, modules = NO_MODULES) -> str | None: + """The call's name if it reads text without an encoding, else None.""" + func = call.func + if isinstance(func, ast.Attribute): + receiver = func.value.id if isinstance(func.value, ast.Name) else None + # An unbound `Path.read_text(p)` puts the instance in slot 0, and + # `pathlib.Path.read_text(p)` is the same call fully qualified. + shift = 1 if _is_path_class(receiver, modules) or _is_path_attr(func.value) else 0 + if func.attr in GUARDED_METHODS: + if func.attr == "read_text" and not shift and call.args: + first = call.args[0] + # Bound read_text takes encoding first, so None or "locale" + # there is a platform-default read. Any other positional means + # the receiver is importlib.metadata's Distribution, whose + # argument is a filename and which takes no encoding at all. + if isinstance(first, ast.Constant) and first.value in PLATFORM_DEFAULT_ENCODINGS: + return "read_text()" + return None + position = ENCODING_POSITION[func.attr] + shift + return None if _pins_encoding(call, position) else f"{func.attr}()" + if func.attr == "open": + # io.open and builtins.open ARE the builtin, so they take the + # builtin's argument positions and the same platform default. + if receiver is not None and _origin_root(receiver, modules) in BUILTIN_OPEN_MODULES: + if not _is_text(call, 1) or _pins_encoding(call, ENCODING_POSITION["open"]): + return None + return f"{receiver}.open()" + compressed = _compressed_key(receiver, modules) if receiver else None + if compressed is not None: + mode = _open_mode(call, 1) + if mode is UNKNOWN_MODE or "t" not in str(mode): + return None # "rb" default, so binary unless asked otherwise + return ( + None + if _pins_encoding(call, COMPRESSED_OPENERS[compressed]) + else f"{compressed}.open()" + ) + # Any other module receiver is somebody else's opener: tarfile.open + # takes a compression mode, Image.open takes a binary file. Neither + # has an encoding to name, so demanding one leaves no correct edit. + if ( + receiver is not None + and receiver in modules + and not _is_path_class(receiver, modules) + ): + return None + if not _is_text(call, shift): + return None + return ( + None + if _pins_encoding(call, ENCODING_POSITION["Path.open"] + shift) + else "Path.open()" + ) + return None + if isinstance(func, ast.Name): + alias = _open_alias(func.id, modules) + # Binary handles have no encoding to name. + if alias == "builtin" and _is_text(call, 1): + return None if _pins_encoding(call, ENCODING_POSITION["open"]) else "open()" + if alias is not None and alias != "builtin": + mode = _open_mode(call, 1) + if mode is UNKNOWN_MODE or "t" not in str(mode): + return None # "rb" default, so binary unless asked otherwise + position = COMPRESSED_OPENERS[alias] + return None if _pins_encoding(call, position) else f"{alias}.open()" + return None + + +def _scan(tree: ast.Module, rel: str): + """Offenders from both rules, reported once each and in source order.""" + modules = _imported_names(tree) + visible_at = _imports_at_each_call(tree) + calls = {id(c): c for c in _import_time_calls(tree)} + calls.update({id(c): c for c in _checked_in_path_calls(tree, modules, visible_at)}) + not_paths = _non_path_names(tree) + temp_roots = _temp_rooted_names(tree) + for call in sorted(calls.values(), key = lambda c: (c.lineno, c.col_offset)): + func = call.func + if ( + isinstance(func, ast.Attribute) + and (func.attr in GUARDED_METHODS or func.attr == "open") + and isinstance(func.value, ast.Name) + and func.value.id in not_paths + ): + continue # ZipFile.open and friends have no encoding to name + expr = _path_expr(call, visible_at.get(id(call), modules)) + root = _path_root(expr) if expr is not None else None + if isinstance(root, ast.Name) and root.id in temp_roots: + continue # the run made this file, so the platform default is safe + name = _offender(call, visible_at.get(id(call), modules)) + if name is not None: + yield f"{rel}:{call.lineno}: {name}" + + +def test_checked_in_file_reads_name_an_encoding(): + offenders = [] + for path in sorted(SOURCES): + tree = ast.parse(path.read_text(encoding = "utf-8"), filename = str(path)) + offenders.extend(_scan(tree, path.relative_to(REPO).as_posix())) + assert offenders == [], ( + f"{len(offenders)} file reads in the test trees touch a checked-in file " + "with the platform default encoding, so they break on Windows as soon " + 'as that file gains a non-ASCII byte. Pass encoding = "utf-8": ' + f"{offenders[:10]}" + ) diff --git a/tests/test_studio_install_workspace_guard.py b/tests/test_studio_install_workspace_guard.py index 18678a9b4a..fa6c8afea4 100644 --- a/tests/test_studio_install_workspace_guard.py +++ b/tests/test_studio_install_workspace_guard.py @@ -15,16 +15,13 @@ SETUP_SH = REPO_ROOT / "studio" / "setup.sh" # Stubs for helpers the extracted guard block calls; mv-based replacement reproduces the venv-gone # effect without the full rollback machinery. _INSTALL_GUARD_STUBS = ( - "substep() { :; }\n" - "_start_studio_venv_replacement() {\n" - ' mv -- "$1" "$1.replaced"\n' - "}\n" + 'substep() { :; }\n_start_studio_venv_replacement() {\n mv -- "$1" "$1.replaced"\n}\n' ) def _extract_install_sh_guard_block() -> str: """Extract install.sh's venv guard block (up to the first elif) as a self-contained snippet.""" - src = INSTALL_SH.read_text() + src = INSTALL_SH.read_text(encoding = "utf-8") m = re.search( r'(if \[ -x "\$VENV_DIR/bin/python" \]; then\n.*?)elif \[ "\$_STUDIO_HOME_REDIRECT" != "env"', src, @@ -119,7 +116,7 @@ def test_default_mode_skips_sentinel_check(tmp_path): def test_install_ps1_has_matching_env_mode_guard(): - src = INSTALL_PS1.read_text() + src = INSTALL_PS1.read_text(encoding = "utf-8") block_start = src.index("if (Test-Path -LiteralPath $VenvPython)") block = src[block_start : block_start + 2000] assert ( @@ -131,7 +128,7 @@ def test_install_ps1_has_matching_env_mode_guard(): def test_setup_ps1_has_writability_probe(): - src = SETUP_PS1.read_text() + src = SETUP_PS1.read_text(encoding = "utf-8") idx = src.index("if (Test-Path -LiteralPath $_studioOverride -PathType Container)") block = src[idx : idx + 2000] assert ( @@ -193,7 +190,7 @@ def test_env_mode_passes_when_bin_unsloth_is_a_symlink(tmp_path): def test_install_ps1_sentinel_uses_pathtype_leaf(): """Remove-Item $VenvDir gate must use -PathType Leaf so a sentinel-path directory cannot satisfy it.""" - src = INSTALL_PS1.read_text() + src = INSTALL_PS1.read_text(encoding = "utf-8") block_start = src.index("if (Test-Path -LiteralPath $VenvPython)") block = src[block_start : block_start + 2000] assert ( @@ -206,7 +203,7 @@ def test_install_ps1_sentinel_uses_pathtype_leaf(): def test_setup_ps1_stale_venv_has_env_mode_guard(): """setup.ps1 stale-venv branch must gate Remove-Item $VenvDir on a custom-root Unsloth sentinel.""" - src = SETUP_PS1.read_text() + src = SETUP_PS1.read_text(encoding = "utf-8") idx = src.index("Stale venv detected") block = src[idx : idx + 1500] assert ( @@ -226,7 +223,7 @@ def test_setup_ps1_stale_venv_has_env_mode_guard(): def test_setup_sh_prebuilt_llama_cpp_has_ownership_guard(): """setup.sh prebuilt llama.cpp path must _assert_studio_owned_or_absent before install_llama_prebuilt.py.""" - src = SETUP_SH.read_text() + src = SETUP_SH.read_text(encoding = "utf-8") idx = src.index("installing prebuilt llama.cpp...") block = src[idx : idx + 2000] assert ( @@ -240,7 +237,7 @@ def test_setup_sh_prebuilt_llama_cpp_has_ownership_guard(): def test_setup_ps1_prebuilt_llama_cpp_has_ownership_guard(): """setup.ps1 prebuilt llama.cpp path must Assert-StudioOwnedOrAbsent before install_llama_prebuilt.py.""" - src = SETUP_PS1.read_text() + src = SETUP_PS1.read_text(encoding = "utf-8") idx = src.index("installing prebuilt llama.cpp bundle (preferred path)") block = src[idx : idx + 2000] assert ( @@ -266,9 +263,9 @@ def test_env_mode_passes_when_venv_marker_present(tmp_path): """install.sh env-mode guard must accept the in-VENV .unsloth-studio-owned marker as a sentinel.""" studio_home = tmp_path / "ws" res = _run_install_guard(studio_home, redirect = "env", create_venv_marker = True) - assert res.returncode == 0, ( - f"in-VENV marker must allow cleanup; " f"stdout={res.stdout!r} stderr={res.stderr!r}" - ) + assert ( + res.returncode == 0 + ), f"in-VENV marker must allow cleanup; stdout={res.stdout!r} stderr={res.stderr!r}" assert "RESULT=ok" in res.stdout assert not (studio_home / "unsloth_studio").exists() @@ -318,16 +315,15 @@ def test_env_mode_blocks_when_bin_unsloth_is_broken_symlink(tmp_path): text = True, capture_output = True, ) - assert res.returncode != 0, ( - "broken symlink at bin/unsloth must NOT pass; " - f"stdout={res.stdout!r} stderr={res.stderr!r}" - ) + assert ( + res.returncode != 0 + ), f"broken symlink at bin/unsloth must NOT pass; stdout={res.stdout!r} stderr={res.stderr!r}" assert (venv / "important.txt").is_file() def test_install_sh_writes_venv_marker_after_uv_venv(): """install.sh must write .unsloth-studio-owned into $VENV_DIR right after `uv venv` succeeds.""" - src = INSTALL_SH.read_text() + src = INSTALL_SH.read_text(encoding = "utf-8") create_idx = src.index('run_install_cmd "create venv" uv venv "$VENV_DIR"') tail = src[create_idx : create_idx + 600] assert ( @@ -337,7 +333,7 @@ def test_install_sh_writes_venv_marker_after_uv_venv(): def test_install_ps1_writes_venv_marker_after_uv_venv(): """install.ps1 must write .unsloth-studio-owned into $VenvDir after `uv venv` succeeds.""" - src = INSTALL_PS1.read_text() + src = INSTALL_PS1.read_text(encoding = "utf-8") venv_create = src.index("uv venv $VenvDir --python") tail = src[venv_create : venv_create + 1500] assert ( @@ -347,7 +343,7 @@ def test_install_ps1_writes_venv_marker_after_uv_venv(): def test_install_ps1_guard_accepts_venv_marker(): """install.ps1 env-mode guard must accept the in-VENV .unsloth-studio-owned marker as a sentinel.""" - src = INSTALL_PS1.read_text() + src = INSTALL_PS1.read_text(encoding = "utf-8") block_start = src.index("if (Test-Path -LiteralPath $VenvPython)") block = src[block_start : block_start + 2000] assert ( @@ -357,7 +353,7 @@ def test_install_ps1_guard_accepts_venv_marker(): def test_setup_helpers_gate_on_canonical_custom_root(): """setup.sh/setup.ps1 ownership guards must gate on a canonical custom-vs-legacy root comparison.""" - sh_src = SETUP_SH.read_text() + sh_src = SETUP_SH.read_text(encoding = "utf-8") sh_idx = sh_src.index("_assert_studio_owned_or_absent() {") sh_func = sh_src[sh_idx : sh_idx + 600] assert ( @@ -369,7 +365,7 @@ def test_setup_helpers_gate_on_canonical_custom_root(): and "_STUDIO_HOME_IS_CUSTOM=" in sh_src ), "setup.sh must compute the canonical custom-root flag" - ps_src = SETUP_PS1.read_text() + ps_src = SETUP_PS1.read_text(encoding = "utf-8") ps_idx = ps_src.index("function Assert-StudioOwnedOrAbsent") ps_func = ps_src[ps_idx : ps_idx + 800] assert ( @@ -382,7 +378,7 @@ def test_setup_helpers_gate_on_canonical_custom_root(): def test_setup_ps1_inplace_git_sync_marks_studio_owned(): """setup.ps1 in-place git-sync branch must Mark-StudioOwned after a successful sync.""" - src = SETUP_PS1.read_text() + src = SETUP_PS1.read_text(encoding = "utf-8") inplace_idx = src.index('Test-Path -LiteralPath (Join-Path $LlamaCppDir ".git")') # The in-place branch ends just before the temp-dir clone branch. clone_idx = src.index("Cloning llama.cpp @", inplace_idx) @@ -397,7 +393,7 @@ def test_setup_ps1_inplace_git_sync_marks_studio_owned(): def test_setup_ps1_inplace_git_sync_asserts_studio_owned_before_mutation(): """setup.ps1 in-place git-sync must Assert-StudioOwnedOrAbsent before any destructive git op.""" - src = SETUP_PS1.read_text() + src = SETUP_PS1.read_text(encoding = "utf-8") inplace_idx = src.index('Test-Path -LiteralPath (Join-Path $LlamaCppDir ".git")') clone_idx = src.index("Cloning llama.cpp @", inplace_idx) inplace_block = src[inplace_idx:clone_idx] @@ -410,7 +406,7 @@ def test_setup_ps1_inplace_git_sync_asserts_studio_owned_before_mutation(): def _extract_check_health_function() -> str: - src = INSTALL_SH.read_text() + src = INSTALL_SH.read_text(encoding = "utf-8") fn_start = src.index("_check_health() {") fn_end = src.index("\n}\n", fn_start) + 2 return src[fn_start:fn_end] @@ -498,7 +494,7 @@ def test_check_health_handles_arbitrary_id_token(): def test_install_ps1_test_studio_health_verifies_studio_root_id(): """install.ps1 Test-StudioHealth must compare studio_root_id against baked $_ExpectedStudioRootId.""" - src = INSTALL_PS1.read_text() + src = INSTALL_PS1.read_text(encoding = "utf-8") fn_start = src.index("function Test-StudioHealth") fn_end = src.index("\n}\n", fn_start) + 2 fn = src[fn_start:fn_end] @@ -510,7 +506,7 @@ def test_install_ps1_test_studio_health_verifies_studio_root_id(): def test_install_ps1_bakes_studio_root_id_into_launcher(): """install.ps1 must persist a CSPRNG id at share/studio_install_id and bake it as $_ExpectedStudioRootId.""" - src = INSTALL_PS1.read_text() + src = INSTALL_PS1.read_text(encoding = "utf-8") assert "$_studioRootId" in src, "install.ps1 must compute $_studioRootId for the launcher" assert ( '"share"' in src and "studio_install_id" in src @@ -526,7 +522,7 @@ def test_install_ps1_bakes_studio_root_id_into_launcher(): def test_health_endpoint_exposes_studio_root_id_not_raw_path(): """/api/health must expose studio_root_id (hex digest), NOT the raw path (info disclosure on -H 0.0.0.0).""" main_py = REPO_ROOT / "studio" / "backend" / "main.py" - src = main_py.read_text() + src = main_py.read_text(encoding = "utf-8") health_idx = src.index('@app.get("/api/health")') # Slice up to the next top-level @app. so a growing body stays in scope. next_app_idx = src.find("\n@app.", health_idx + 1) @@ -542,7 +538,7 @@ def test_health_endpoint_exposes_studio_root_id_not_raw_path(): def test_install_sh_bakes_studio_root_id_into_launcher(): """install.sh must persist the id at share/studio_install_id and bake it into the launcher for ALL modes.""" - src = INSTALL_SH.read_text() + src = INSTALL_SH.read_text(encoding = "utf-8") assert ( "_css_studio_root_id" in src ), "install.sh must compute _css_studio_root_id for the launcher" @@ -568,8 +564,10 @@ def test_tauri_preflight_scrubs_studio_home_env(): preflight_root / "preflight.rs", *(preflight_root / "preflight").glob("*.rs"), ] - preflight = "\n".join(p.read_text() for p in preflight_paths if p.exists()) - commands = (REPO_ROOT / "studio" / "src-tauri" / "src" / "commands.rs").read_text() + preflight = "\n".join(p.read_text(encoding = "utf-8") for p in preflight_paths if p.exists()) + commands = (REPO_ROOT / "studio" / "src-tauri" / "src" / "commands.rs").read_text( + encoding = "utf-8" + ) # Expect 2 scrubs in preflight (run_cli_probe + probe_cli_capability), 1 in commands. assert ( preflight.count('cmd.env_remove("UNSLOTH_STUDIO_HOME")') >= 2 @@ -587,7 +585,7 @@ def test_tauri_preflight_scrubs_studio_home_env(): def test_install_sh_shim_uses_atomic_replace(): """install.sh shim install must use ln -sfn for atomic replace (rm+ln left a missing-shim window).""" - src = INSTALL_SH.read_text() + src = INSTALL_SH.read_text(encoding = "utf-8") shim_idx = src.index('_shim_path="$_LOCAL_BIN/unsloth"') block = src[shim_idx : shim_idx + 1500] assert ( @@ -600,7 +598,7 @@ def test_install_sh_shim_uses_atomic_replace(): def test_install_sh_create_shortcuts_seeds_id_from_csprng_with_python_fallback(tmp_path): """_create_shortcuts seeds ids from /dev/urandom (python3 secrets fallback) and is re-run idempotent.""" - src = INSTALL_SH.read_text() + src = INSTALL_SH.read_text(encoding = "utf-8") fn_start = src.index('_css_data_dir="$DATA_DIR"') block = src[fn_start : fn_start + 3000] urandom_idx = block.index("od -An -N32 -tx1 /dev/urandom") @@ -645,7 +643,7 @@ def test_install_sh_create_shortcuts_seeds_id_from_csprng_with_python_fallback(t def test_install_sh_create_shortcuts_fails_fast_when_no_entropy(): """With no entropy source, _create_shortcuts must `return 1` not bake an empty studio_root_id.""" - src = INSTALL_SH.read_text() + src = INSTALL_SH.read_text(encoding = "utf-8") fn_start = src.index('_css_data_dir="$DATA_DIR"') block = src[fn_start : fn_start + 3000] assert ( @@ -661,7 +659,7 @@ def test_install_sh_create_shortcuts_fails_fast_when_no_entropy(): def test_install_sh_bakes_installed_is_env_mode_flag_in_launcher(): """install.sh must bake the install-time mode into the launcher so a sourced studio.conf can't flip it.""" - src = INSTALL_SH.read_text() + src = INSTALL_SH.read_text(encoding = "utf-8") assert ( "_INSTALLED_IS_ENV_MODE='@@INSTALLED_IS_ENV_MODE@@'" in src ), "launcher heredoc must declare _INSTALLED_IS_ENV_MODE='@@INSTALLED_IS_ENV_MODE@@'" @@ -676,7 +674,7 @@ def test_install_sh_bakes_installed_is_env_mode_flag_in_launcher(): def test_install_sh_launcher_gates_port_file_on_baked_flag_not_runtime_env(): """Launcher PORT_FILE/LOCK_DIR must gate on baked $_INSTALLED_IS_ENV_MODE, not runtime $UNSLOTH_STUDIO_HOME.""" - src = INSTALL_SH.read_text() + src = INSTALL_SH.read_text(encoding = "utf-8") heredoc_start = src.index("cat > \"$_css_launcher\" << 'LAUNCHER_EOF'") heredoc_end = src.index("LAUNCHER_EOF\n", heredoc_start) heredoc = src[heredoc_start:heredoc_end] @@ -724,7 +722,7 @@ def test_install_sh_launcher_gates_port_file_on_baked_flag_not_runtime_env(): def test_main_py_studio_root_id_caches_at_module_load(): """_studio_root_id() must read the id once at module load and reuse it (no per-poll FS/hash work).""" - main_py = (REPO_ROOT / "studio" / "backend" / "main.py").read_text() + main_py = (REPO_ROOT / "studio" / "backend" / "main.py").read_text(encoding = "utf-8") assert ( "_STUDIO_ROOT_ID_CACHE: str = _read_studio_install_id()" in main_py ), "main.py must populate _STUDIO_ROOT_ID_CACHE from _read_studio_install_id() at module load" @@ -785,7 +783,7 @@ def test_llama_cpp_search_roots_handles_studio_root_oserror(): holds the handler so the two never disagree on which root is legacy.""" llama_cpp = ( REPO_ROOT / "studio" / "backend" / "core" / "inference" / "llama_cpp.py" - ).read_text() + ).read_text(encoding = "utf-8") def _method_body(name: str) -> str: # Whole method body (def to next sibling def) so the check survives growth. @@ -830,7 +828,7 @@ def test_install_sh_install_id_survives_symlinked_studio_home(tmp_path): def test_install_sh_substitutes_root_id_before_data_dir(): """sed must bake the non-user-controlled placeholders before @@DATA_DIR@@ so a crafted $DATA_DIR isn't mutated.""" - src = INSTALL_SH.read_text() + src = INSTALL_SH.read_text(encoding = "utf-8") root_id_idx = src.index("s|@@STUDIO_ROOT_ID@@|$_css_studio_root_id|g") env_mode_idx = src.index("s|@@INSTALLED_IS_ENV_MODE@@|$_css_is_env_mode|g") data_dir_idx = src.index("s|@@DATA_DIR@@|$_sed_safe|g") @@ -845,13 +843,15 @@ def test_install_sh_substitutes_root_id_before_data_dir(): def test_install_sh_root_id_pass_does_not_mutate_user_data_dir(tmp_path): """A $DATA_DIR containing the literal @@STUDIO_ROOT_ID@@ must survive the placeholder-first sed passes.""" - src = INSTALL_SH.read_text() + src = INSTALL_SH.read_text(encoding = "utf-8") heredoc_start = src.index("cat > \"$_css_launcher\" << 'LAUNCHER_EOF'") heredoc_body_start = src.index("\n", heredoc_start) + 1 heredoc_body_end = src.index("LAUNCHER_EOF\n", heredoc_start) template = src[heredoc_body_start:heredoc_body_end] launcher_path = tmp_path / "launch.sh" - launcher_path.write_text(template) + # template comes out of install.sh, so it carries whatever non-ASCII that + # file holds and cp1252 cannot encode it back out. + launcher_path.write_text(template, encoding = "utf-8") # sed order: root-id first, then data-dir. weird_data_dir = "/tmp/with-@@STUDIO_ROOT_ID@@/share" root_id = "deadbeef" * 8 @@ -866,7 +866,8 @@ sed "s|@@DATA_DIR@@|$_sed_safe|g" "{launcher_path}" > "{launcher_path}.tmp" \\ && mv "{launcher_path}.tmp" "{launcher_path}" """ subprocess.run(["bash", "-c", script], check = True) - final = launcher_path.read_text() + # written as utf-8 just above, and the template carries U+2500. + final = launcher_path.read_text(encoding = "utf-8") assert ( f"DATA_DIR='{weird_data_dir}'" in final ), f"DATA_DIR must be preserved verbatim (no @@STUDIO_ROOT_ID@@ mutation); got: {final[:500]}" @@ -877,7 +878,7 @@ sed "s|@@DATA_DIR@@|$_sed_safe|g" "{launcher_path}" > "{launcher_path}.tmp" \\ def test_install_ps1_install_id_file_layout_matches_backend_read_path(): """install.ps1 must write the id at share/studio_install_id where the backend reads it, idempotently.""" - src = INSTALL_PS1.read_text() + src = INSTALL_PS1.read_text(encoding = "utf-8") id_idx = src.index('$_studioIdDir = Join-Path $StudioHome "share"') context = src[id_idx : id_idx + 1500] assert ( diff --git a/tests/test_studio_root_resilience.py b/tests/test_studio_root_resilience.py index 779ff2f3f1..66195d11fe 100644 --- a/tests/test_studio_root_resilience.py +++ b/tests/test_studio_root_resilience.py @@ -64,7 +64,7 @@ def test_kill_orphan_catches_oserror_from_studio_root(): """Cleanup must not crash when studio_root() raises. _kill_orphaned_servers resolves the install root through the shared _resolved_studio_root_and_is_legacy() classifier, which swallows (ImportError, OSError, ValueError) on the probe.""" - src = LLAMA_CPP.read_text() + src = LLAMA_CPP.read_text(encoding = "utf-8") # Cleanup delegates to the shared classifier rather than importing studio_root inline. assert "LlamaCppBackend._resolved_studio_root_and_is_legacy()" in _method_body( src, "_kill_orphaned_servers" @@ -85,7 +85,7 @@ def _exec_search_roots_block( """Run _find_llama_server_binary's search_roots derivation -- plus the shared _resolved_studio_root_and_is_legacy() classifier it delegates to -- with a controlled studio_root() and resolve(), without importing the heavy module.""" - src = LLAMA_CPP.read_text() + src = LLAMA_CPP.read_text(encoding = "utf-8") # Shared root classifier (holds the defensive try/except for studio_root()). # End the slice at the next sibling def/decorator at the same indent rather # than the literal "@staticmethod" string, so a future docstring mentioning a diff --git a/tests/test_tool_mask_zoo_compat.py b/tests/test_tool_mask_zoo_compat.py index 6212b6f807..84c032da3b 100644 --- a/tests/test_tool_mask_zoo_compat.py +++ b/tests/test_tool_mask_zoo_compat.py @@ -15,7 +15,7 @@ RL_REPLACEMENTS_SOURCE_PATH = os.path.join(REPO_ROOT, "unsloth", "models", "rl_r def _read(path: str) -> str: - with open(path, "r") as fh: + with open(path, "r", encoding = "utf-8") as fh: return fh.read() diff --git a/tests/utils/test_prepare_inputs_leftpad.py b/tests/utils/test_prepare_inputs_leftpad.py index 2bfd763279..9a64103770 100644 --- a/tests/utils/test_prepare_inputs_leftpad.py +++ b/tests/utils/test_prepare_inputs_leftpad.py @@ -46,7 +46,7 @@ WIRED_MODEL_FILES = [ def _load_function(): - tree = ast.parse(LLAMA_PY.read_text()) + tree = ast.parse(LLAMA_PY.read_text(encoding = "utf-8")) for node in ast.walk(tree): if isinstance(node, ast.FunctionDef) and node.name == FUNC_NAME: return node @@ -207,7 +207,7 @@ def test_model_families_stay_wired_to_shared_prepare_inputs(): path = REPO_ROOT / "unsloth" / "models" / fname if not path.exists(): continue - if "fix_prepare_inputs_for_generation(" not in path.read_text(): + if "fix_prepare_inputs_for_generation(" not in path.read_text(encoding = "utf-8"): missing.append(fname) assert not missing, ( "these model files no longer call fix_prepare_inputs_for_generation, " diff --git a/tests/utils/test_rope_scaling_drift.py b/tests/utils/test_rope_scaling_drift.py index b2ec1e5a20..eba89734f7 100644 --- a/tests/utils/test_rope_scaling_drift.py +++ b/tests/utils/test_rope_scaling_drift.py @@ -52,7 +52,7 @@ MAX_POS = 131072 def _load_class_init(): - tree = ast.parse(LLAMA_PY.read_text()) + tree = ast.parse(LLAMA_PY.read_text(encoding = "utf-8")) for node in ast.walk(tree): if isinstance(node, ast.ClassDef) and node.name == CLASS_NAME: for sub in node.body: @@ -96,7 +96,7 @@ def _iter_names_and_calls(node): def _find_method(source_path, class_name, method_name): - for node in ast.walk(ast.parse(source_path.read_text())): + for node in ast.walk(ast.parse(source_path.read_text(encoding = "utf-8"))): if isinstance(node, ast.ClassDef) and node.name == class_name: for sub in node.body: if isinstance(sub, ast.FunctionDef) and sub.name == method_name: @@ -105,7 +105,7 @@ def _find_method(source_path, class_name, method_name): def _find_function(source_path, function_name): - for node in ast.walk(ast.parse(source_path.read_text())): + for node in ast.walk(ast.parse(source_path.read_text(encoding = "utf-8"))): if isinstance(node, ast.FunctionDef) and node.name == function_name: return node return None diff --git a/unsloth_cli/__init__.py b/unsloth_cli/__init__.py index 121b26f03f..703a6f1f60 100644 --- a/unsloth_cli/__init__.py +++ b/unsloth_cli/__init__.py @@ -4,6 +4,28 @@ import os as _os import sys as _sys +# Are we the `unsloth` console script, rather than a library import? Both the +# stream guard below and the `-np` rewrite further down are entry-point +# behaviour and must not reach into a host application that imports us. +_entry_base = _os.path.basename(_sys.argv[0]).lower() if _sys.argv else "" +_is_entry_point = _entry_base in {"unsloth", "unsloth.exe"} + +# Typer renders help via rich, whose box characters cp1252 and cp437 cannot encode, +# so `unsloth --help` dies once stdout is a pipe or a file. Windows gets UTF-8, as +# unsloth/__init__ already does; elsewhere the caller's encoding is kept and only +# the error handler is relaxed, so an explicit PYTHONIOENCODING still picks the +# bytes and only loses unencodable glyphs. Before typer, which binds the stream. +if _is_entry_point: + _to_utf8 = _sys.platform == "win32" + for _name in ("stdout", "stderr"): + _stream = getattr(_sys, _name, None) + try: + if "utf" not in (_stream.encoding or "").lower(): + _stream.reconfigure(encoding = "utf-8" if _to_utf8 else None, errors = "replace") + except Exception: + pass + del _name, _stream, _to_utf8 + import typer from importlib.metadata import version as package_version, PackageNotFoundError @@ -22,10 +44,9 @@ from unsloth_cli.commands.studio import ( # Canonicalise `-np` only under the `unsloth` console-script; # third-party scripts that import unsloth_cli keep their argv intact. -_entry_base = _os.path.basename(_sys.argv[0]).lower() if _sys.argv else "" -if _entry_base in {"unsloth", "unsloth.exe"}: +if _is_entry_point: _expand_attached_np_short() -del _entry_base +del _entry_base, _is_entry_point def show_version(value: bool): diff --git a/unsloth_cli/tests/test_start.py b/unsloth_cli/tests/test_start.py index 4cdb37e7fa..dd0edfaa15 100644 --- a/unsloth_cli/tests/test_start.py +++ b/unsloth_cli/tests/test_start.py @@ -587,7 +587,9 @@ def test_write_codex_config_profile(tmp_path, monkeypatch): assert catalog["models"][0]["supports_reasoning_summary_parameter"] is False assert catalog["models"][0]["supports_parallel_tool_calls"] is False - assert catalog["models"][0]["base_instructions"] == start._CODEX_FALLBACK_PROMPT.read_text() + assert catalog["models"][0]["base_instructions"] == start._CODEX_FALLBACK_PROMPT.read_text( + encoding = "utf-8" + ) config = _parse_toml((tmp_path / "config.toml").read_text()) assert config["model_providers"]["unsloth_api"]["env_key"] == "UNSLOTH_STUDIO_AUTH_TOKEN" @@ -631,7 +633,7 @@ def test_write_codex_subagent_bridge_keeps_parent_credentials_out(tmp_path, monk tmp_path, yolo = False, ) - assert json.loads(path.read_text()) == { + assert json.loads(path.read_text(encoding = "utf-8")) == { "api_key": "private-token", "codex_home": str(tmp_path / "child"), "bypass_permissions": False,