Pin utf-8 on shipping-code text I/O instead of the operator locale (#7486)

* Pin utf-8 on shipping-code text I/O instead of the operator locale

113 read_text/write_text/open call sites across unsloth, studio and
unsloth_cli let locale.getencoding() decide the encoding. That is utf-8 on
the Linux and macOS runners and cp1252 on a stock Windows install, so the
same file decodes differently for a Windows user and silently produces
mojibake or raises UnicodeDecodeError.

Adds tests/test_runtime_text_encoding.py to keep it that way. It resolves
openers through each file's own imports rather than a fixed list of module
names, so an aliased tarfile.open or a local from PIL.Image import open is
not asked for an encoding it does not take.

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Scan tracked files only and resolve the unbound Path calling forms

* Honour PEP 263 when scanning sources and migrate a legacy JSONL before appending

* Scope guard imports lexically and only migrate a legacy file when it round-trips

* Leave a legacy JSONL untouched and resolve path aliases in the foreign-opener check

* Tighten comments

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

studio/backend/auth/storage.py

@@ -44,7 +44,7 @@ def generate_bootstrap_password() -> str:
 
     # Persisted from a previous run?
     if _BOOTSTRAP_PW_PATH.is_file():
-        _bootstrap_password = _BOOTSTRAP_PW_PATH.read_text().strip()
+        _bootstrap_password = _BOOTSTRAP_PW_PATH.read_text(encoding = "utf-8").strip()
         if _bootstrap_password:
             return _bootstrap_password
 
@@ -57,7 +57,7 @@ def generate_bootstrap_password() -> str:
 
     # Persist so the same passphrase survives restarts until password change.
     ensure_dir(_BOOTSTRAP_PW_PATH.parent)
-    _BOOTSTRAP_PW_PATH.write_text(_bootstrap_password)
+    _BOOTSTRAP_PW_PATH.write_text(_bootstrap_password, encoding = "utf-8")
     try:
         os.chmod(_BOOTSTRAP_PW_PATH, 0o600)
     except OSError:
@@ -76,7 +76,7 @@ def _load_bootstrap_password() -> Optional[str]:
     global _bootstrap_password
     _bootstrap_password = None
     if _BOOTSTRAP_PW_PATH.is_file():
-        bootstrap_password = _BOOTSTRAP_PW_PATH.read_text().strip()
+        bootstrap_password = _BOOTSTRAP_PW_PATH.read_text(encoding = "utf-8").strip()
         if bootstrap_password:
             _bootstrap_password = bootstrap_password
     return _bootstrap_password
@@ -99,7 +99,7 @@ def clear_bootstrap_password() -> None:
             # stale plaintext can't be re-seeded by generate_bootstrap_password()
             # if a later reset-password deletes auth.db and re-validates it.
             try:
-                _BOOTSTRAP_PW_PATH.write_text("")
+                _BOOTSTRAP_PW_PATH.write_text("", encoding = "utf-8")
                 cleared = True
             except OSError:
                 cleared = False

studio/backend/colab.py

@@ -90,7 +90,7 @@ def _store_colab_login_credentials(username: str, password: str) -> None:
     path = _colab_login_credentials_path()
     try:
         path.parent.mkdir(parents = True, exist_ok = True)
-        path.write_text(f"{username}\n{password}\n")
+        path.write_text(f"{username}\n{password}\n", encoding = "utf-8")
         try:
             import os
             os.chmod(path, 0o600)
@@ -106,7 +106,7 @@ def _load_colab_login_credentials() -> "tuple[str, str] | None":
     try:
         if not path.is_file():
             return None
-        lines = path.read_text().splitlines()
+        lines = path.read_text(encoding = "utf-8").splitlines()
         if len(lines) >= 2 and lines[0] and lines[1]:
             return lines[0], lines[1]
     except OSError as e:

studio/backend/core/export/export.py

@@ -241,7 +241,7 @@ def _offline_window_if(local_files_only):
 def _is_wsl():
     """Detect if running under Windows Subsystem for Linux."""
     try:
-        return "microsoft" in open("/proc/version").read().lower()
+        return "microsoft" in open("/proc/version", encoding = "utf-8").read().lower()
     except Exception:
         return False
 
@@ -574,7 +574,7 @@ class ExportBackend:
             )
             metadata = {"base_model": base_model}
             metadata_path = os.path.join(save_directory, "export_metadata.json")
-            with open(metadata_path, "w") as f:
+            with open(metadata_path, "w", encoding = "utf-8") as f:
                 json.dump(metadata, f, indent = 2)
             logger.info(f"Wrote export metadata to {metadata_path}")
         except Exception as e:

studio/backend/core/inference/inference.py

@@ -567,7 +567,7 @@ class InferenceBackend:
                         _meta_path = Path(config.path) / "export_metadata.json"
                         try:
                             if _meta_path.exists():
-                                _meta = json.loads(_meta_path.read_text())
+                                _meta = json.loads(_meta_path.read_text(encoding = "utf-8"))
                                 if _meta.get("base_model"):
                                     processor_source = _meta["base_model"]
                         except Exception:

studio/backend/core/inference/llama_cpp.py

@@ -569,7 +569,7 @@ def _load_swa_cache() -> dict:
         if _SWA_CACHE is not None:
             return _SWA_CACHE
         try:
-            with open(_swa_cache_path()) as f:
+            with open(_swa_cache_path(), encoding = "utf-8") as f:
                 _SWA_CACHE = json.load(f)
                 if not isinstance(_SWA_CACHE, dict):
                     _SWA_CACHE = {}
@@ -583,7 +583,7 @@ def _save_swa_cache(cache: dict) -> None:
         path = _swa_cache_path()
         path.parent.mkdir(parents = True, exist_ok = True)
         tmp = path.with_suffix(".json.tmp")
-        with open(tmp, "w") as f:
+        with open(tmp, "w", encoding = "utf-8") as f:
             json.dump(cache, f, indent = 2, sort_keys = True)
         tmp.replace(path)
     except OSError:
@@ -620,7 +620,7 @@ def _fetch_swa_entry_from_hf(repo_id: str) -> Optional[object]:
             repo_type = "model",
             cache_dir = active_hf_hub_cache(),
         )
-        with open(cfg_path) as f:
+        with open(cfg_path, encoding = "utf-8") as f:
             cfg = json.load(f)
     except Exception:
         return None
@@ -3596,7 +3596,7 @@ class LlamaCppBackend:
         except Exception:
             pass
         try:
-            with open("/proc/meminfo") as f:
+            with open("/proc/meminfo", encoding = "utf-8") as f:
                 for line in f:
                     if line.startswith("MemAvailable:"):
                         return int(line.split()[1]) // 1024  # kB -> MiB
@@ -9477,7 +9477,7 @@ class LlamaCppBackend:
             return
         try:
             path.parent.mkdir(parents = True, exist_ok = True)
-            path.write_text(f"{pid}:{cls._pid_start_identity(pid)}")
+            path.write_text(f"{pid}:{cls._pid_start_identity(pid)}", encoding = "utf-8")
         except Exception as e:
             logger.debug(f"Could not write llama-server pidfile: {e}")
 
@@ -9611,7 +9611,7 @@ class LlamaCppBackend:
         pid = -1
         identity = ""
         try:
-            pid_str, _, identity = path.read_text().strip().partition(":")
+            pid_str, _, identity = path.read_text(encoding = "utf-8").strip().partition(":")
             pid = int(pid_str)
         except Exception:
             pid = -1

studio/backend/core/inference/sandbox_site/sitecustomize.py

@@ -111,7 +111,7 @@ def _load_sidecar(cwd):
     """Return the persisted ``source -> healed target`` map, or {} on any error
     (missing/corrupt/foreign sidecar degrades to in-process-only behaviour)."""
     try:
-        with open(_sidecar_path(cwd)) as fh:
+        with open(_sidecar_path(cwd), encoding = "utf-8") as fh:
             data = json.load(fh)
     except Exception:  # noqa: BLE001 - a bad sidecar must never break user code
         return {}
@@ -131,7 +131,7 @@ def _record_sidecar(cwd, source, target):
             return
         data[source] = target
         tmp = _sidecar_path(cwd) + ".tmp"
-        with open(tmp, "w") as fh:
+        with open(tmp, "w", encoding = "utf-8") as fh:
             json.dump(data, fh)
         os.replace(tmp, _sidecar_path(cwd))
     except Exception:  # noqa: BLE001 - persistence is best effort only

studio/backend/core/inference/worker.py

@@ -151,7 +151,7 @@ def _resolve_lora_4bit(mc, load_in_4bit: bool) -> bool:
     import json
 
     try:
-        with open(adapter_cfg_path) as f:
+        with open(adapter_cfg_path, encoding = "utf-8") as f:
             adapter_cfg = json.load(f)
         training_method = adapter_cfg.get("unsloth_training_method")
         if training_method == "lora" and load_in_4bit:
@@ -961,7 +961,10 @@ def run_inference_process(
     if _local_adapter_cfg.is_file():
         try:
             _lora_base = (
-                _json.loads(_local_adapter_cfg.read_text()).get("base_model_name_or_path") or None
+                _json.loads(_local_adapter_cfg.read_text(encoding = "utf-8")).get(
+                    "base_model_name_or_path"
+                )
+                or None
             )
         except Exception:
             _lora_base = None

studio/backend/core/rag/embeddings.py

@@ -100,7 +100,7 @@ def _st_module_subdirs(name: str, token: str | None) -> tuple[str, ...]:
             path = Path(normalize_path(name)).expanduser() / "modules.json"
             if not path.is_file():
                 return ()
-            data = json.loads(path.read_text())
+            data = json.loads(path.read_text(encoding = "utf-8"))
         else:
             from huggingface_hub import hf_hub_download
             from huggingface_hub.utils import EntryNotFoundError
@@ -115,7 +115,7 @@ def _st_module_subdirs(name: str, token: str | None) -> tuple[str, ...]:
                 )
             except EntryNotFoundError:
                 return ()
-            data = json.loads(open(local).read())
+            data = json.loads(open(local, encoding = "utf-8").read())
         subdirs = []
         for module in data or ():
             sub = str((module or {}).get("path", "")).strip().strip("/")

studio/backend/hub/services/models/ollama.py

@@ -215,7 +215,7 @@ def _ollama_model_info_from_manifest(
         return None
 
     try:
-        manifest = json.loads(tag_file.read_text())
+        manifest = json.loads(tag_file.read_text(encoding = "utf-8"))
     except (json.JSONDecodeError, OSError) as e:
         logger.debug("Skipping unreadable/invalid Ollama manifest %s: %s", tag_file, e)
         return None
@@ -228,7 +228,7 @@ def _ollama_model_info_from_manifest(
         config_blob = _ollama_blob_path(blobs_dir, config_digest)
         if config_blob is not None and _safe_is_file(config_blob):
             try:
-                cfg = json.loads(config_blob.read_text())
+                cfg = json.loads(config_blob.read_text(encoding = "utf-8"))
                 model_type = cfg.get("model_type", "")
                 file_type = cfg.get("file_type", "")
             except (json.JSONDecodeError, OSError) as e:

studio/backend/hub/utils/download_registry.py

@@ -462,7 +462,7 @@ def _read_marker_value(marker: Path) -> Optional[str]:
     try:
         if not marker.exists():
             return None
-        value = marker.read_text().strip()
+        value = marker.read_text(encoding = "utf-8").strip()
     except OSError:
         return None
     return value if value in VALID_TRANSPORTS else None
@@ -473,7 +473,7 @@ def _write_marker_value(marker: Path, mode: str) -> None:
         # tmp + rename so a SIGKILL mid-write can't leave a half-written marker.
         # The tmp name is per-process so concurrent writers don't clobber tmps.
         tmp = marker.with_name(f"{marker.name}.tmp-{os.getpid()}")
-        tmp.write_text(mode)
+        tmp.write_text(mode, encoding = "utf-8")
         os.replace(tmp, marker)
     except OSError:
         # Best-effort: a missing marker next run purges the partial defensively,

studio/backend/hub/utils/paths.py

@@ -103,7 +103,7 @@ def _is_wsl() -> bool:
     if sys.platform == "win32":
         return False
     try:
-        return "microsoft" in Path("/proc/version").read_text().lower()
+        return "microsoft" in Path("/proc/version").read_text(encoding = "utf-8").lower()
     except Exception:
         return False
 
@@ -124,7 +124,7 @@ def _wsl_automount_root() -> str:
         import configparser
 
         parser = configparser.ConfigParser(inline_comment_prefixes = ("#", ";"))
-        parser.read("/etc/wsl.conf")
+        parser.read("/etc/wsl.conf", encoding = "utf-8")
         root = parser.get("automount", "root", fallback = "").strip().strip("\"'")
     except Exception:
         return default

studio/backend/main.py

@@ -254,7 +254,11 @@ def _read_studio_install_id() -> str:
     /api/health emits "" and the launcher accepts any healthy backend.
     Carries no install-path info (matters when Unsloth runs -H 0.0.0.0)."""
     try:
-        token = (_STUDIO_ROOT_RESOLVED / "share" / "studio_install_id").read_text().strip()
+        token = (
+            (_STUDIO_ROOT_RESOLVED / "share" / "studio_install_id")
+            .read_text(encoding = "utf-8")
+            .strip()
+        )
     except (OSError, ValueError):
         return ""
     return token if _STUDIO_INSTALL_ID_RE.fullmatch(token) else ""

studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/state_store.py

@@ -20,7 +20,7 @@ class StateStore:
         self._data: Dict[str, Any] = {}
         if self.path.exists():
             try:
-                with self.path.open() as f:
+                with self.path.open(encoding = "utf-8") as f:
                     self._data = json.load(f)
             except Exception:
                 self._data = {}
@@ -51,7 +51,7 @@ class StateStore:
 
     def _flush(self) -> None:
         tmp = self.path.with_suffix(self.path.suffix + ".tmp")
-        with tmp.open("w") as f:
+        with tmp.open("w", encoding = "utf-8") as f:
             json.dump(self._data, f, indent = 2, default = str)
         os.replace(tmp, self.path)
 
@@ -63,12 +63,14 @@ class JsonlWriter:
         self.path = Path(path)
         self.path.parent.mkdir(parents = True, exist_ok = True)
         self._lock = threading.Lock()
-        self._fh = self.path.open("a", buffering = 1)
+        self._fh = self.path.open("a", buffering = 1, encoding = "utf-8")
         self._count_seen_keys: set[str] = set()
         # Preload seen keys for dedup across resumes
         if self.path.exists() and self.path.stat().st_size > 0:
             try:
-                with self.path.open() as f:
+                # No guess is safe for a file an older build wrote in the
+                # operator's locale, so read past whatever will not decode.
+                with self.path.open(encoding = "utf-8", errors = "replace") as f:
                     for line in f:
                         try:
                             obj = json.loads(line)

studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/impl.py

@@ -27,7 +27,7 @@ class UnstructuredSeedReader(SeedReader[UnstructuredSeedSource]):
             orig_name = path_obj.name
             if meta_path.exists():
                 try:
-                    meta = json_mod.loads(meta_path.read_text())
+                    meta = json_mod.loads(meta_path.read_text(encoding = "utf-8"))
                     orig_name = meta.get("original_filename", path_obj.name)
                 except (json_mod.JSONDecodeError, OSError):
                     pass

studio/backend/routes/inference.py

@@ -3778,7 +3778,7 @@ def _effective_load_in_4bit(config: ModelConfig, requested: bool) -> bool:
     if not adapter_cfg_path.exists():
         return load_in_4bit
     try:
-        with open(adapter_cfg_path) as f:
+        with open(adapter_cfg_path, encoding = "utf-8") as f:
             adapter_cfg = json.load(f)
         if not isinstance(adapter_cfg, dict):  # malformed -> keep requested
             return load_in_4bit

studio/backend/routes/models.py

@@ -722,7 +722,7 @@ def _scan_ollama_dir(ollama_dir: Path, limit: Optional[int] = None) -> List[Loca
             stem_hash = hashlib.sha256(manifest_key.encode()).hexdigest()[:10]
 
             try:
-                manifest = json.loads(tag_file.read_text())
+                manifest = json.loads(tag_file.read_text(encoding = "utf-8"))
             except (json.JSONDecodeError, OSError) as e:
                 logger.debug(
                     "Skipping unreadable/invalid Ollama manifest %s: %s",
@@ -738,7 +738,7 @@ def _scan_ollama_dir(ollama_dir: Path, limit: Optional[int] = None) -> List[Loca
                 config_blob = blobs_dir / config_digest.replace(":", "-")
                 if config_blob.is_file():
                     try:
-                        cfg = json.loads(config_blob.read_text())
+                        cfg = json.loads(config_blob.read_text(encoding = "utf-8"))
                         model_type = cfg.get("model_type", "")
                         file_type = cfg.get("file_type", "")
                     except (json.JSONDecodeError, OSError) as e:
@@ -1042,7 +1042,7 @@ def _dir_has_downloaded_model(directory: Path, max_entries: int = 4000) -> bool:
                 if not m.is_file():
                     continue
                 try:
-                    manifest = json.loads(m.read_text())
+                    manifest = json.loads(m.read_text(encoding = "utf-8"))
                 except (json.JSONDecodeError, OSError, ValueError):
                     continue
                 for layer in manifest.get("layers") or []:

studio/backend/run.py

@@ -774,7 +774,7 @@ def _write_pid_file():
     """Write the current process PID to the studio PID file."""
     try:
         _PID_FILE.parent.mkdir(parents = True, exist_ok = True)
-        _PID_FILE.write_text(str(os.getpid()))
+        _PID_FILE.write_text(str(os.getpid()), encoding = "utf-8")
     except OSError:
         pass
 
@@ -783,7 +783,7 @@ def _remove_pid_file():
     """Remove the PID file if it belongs to this process."""
     try:
         if _PID_FILE.is_file():
-            stored = _PID_FILE.read_text().strip()
+            stored = _PID_FILE.read_text(encoding = "utf-8").strip()
             if stored == str(os.getpid()):
                 _PID_FILE.unlink(missing_ok = True)
     except OSError:

studio/backend/utils/hardware/hardware.py

@@ -776,7 +776,7 @@ def _rocm_linux_sysfs_gpu_busy_pct() -> Optional[float]:
         files = glob.glob("/sys/class/drm/card*/device/gpu_busy_percent")
         if not files:
             return None
-        values = [int(open(f).read().strip()) for f in files]
+        values = [int(open(f, encoding = "utf-8").read().strip()) for f in files]
         return round(sum(values) / len(values), 1)
     except Exception:
         return None
@@ -790,7 +790,7 @@ def _rocm_linux_sysfs_temp_c() -> Optional[float]:
         files = glob.glob("/sys/class/drm/card*/device/hwmon/hwmon*/temp1_input")
         if not files:
             return None
-        temps = [int(open(f).read().strip()) / 1000.0 for f in files]
+        temps = [int(open(f, encoding = "utf-8").read().strip()) / 1000.0 for f in files]
         return round(max(temps), 1)
     except Exception:
         return None
@@ -807,7 +807,9 @@ def _rocm_linux_sysfs_power_w() -> Optional[float]:
         ):
             files = glob.glob(pattern)
             if files:
-                watts = sum(int(open(f).read().strip()) / 1_000_000.0 for f in files)
+                watts = sum(
+                    int(open(f, encoding = "utf-8").read().strip()) / 1_000_000.0 for f in files
+                )
                 return round(watts, 1)
         return None
     except Exception:
@@ -852,8 +854,8 @@ def _rocm_linux_sysfs_vram_gb() -> tuple[Optional[float], Optional[float]]:
         total_files = glob.glob("/sys/class/drm/card*/device/mem_info_vram_total")
         if not used_files or not total_files:
             return None, None
-        used_bytes = sum(int(open(f).read().strip()) for f in used_files)
-        total_bytes = sum(int(open(f).read().strip()) for f in total_files)
+        used_bytes = sum(int(open(f, encoding = "utf-8").read().strip()) for f in used_files)
+        total_bytes = sum(int(open(f, encoding = "utf-8").read().strip()) for f in total_files)
         if total_bytes == 0:
             return None, None
         return round(used_bytes / (1024**3), 2), round(total_bytes / (1024**3), 2)
@@ -893,7 +895,7 @@ def _rocm_kfd_gpu_pci_ids() -> list[str]:
             continue
         props: dict[str, int] = {}
         try:
-            with open(os.path.join(node_dir, "properties")) as f:
+            with open(os.path.join(node_dir, "properties"), encoding = "utf-8") as f:
                 for line in f:
                     parts = line.split()
                     if len(parts) == 2:
@@ -979,9 +981,9 @@ def _rocm_linux_sysfs_vram_by_pci_gb() -> dict[str, tuple[float, float]]:
             if not bdf:
                 continue
             try:
-                with open(os.path.join(dev_dir, "mem_info_vram_used")) as f:
+                with open(os.path.join(dev_dir, "mem_info_vram_used"), encoding = "utf-8") as f:
                     used_bytes = int(f.read().strip())
-                with open(os.path.join(dev_dir, "mem_info_vram_total")) as f:
+                with open(os.path.join(dev_dir, "mem_info_vram_total"), encoding = "utf-8") as f:
                     total_bytes = int(f.read().strip())
             except (OSError, ValueError):
                 continue

studio/backend/utils/models/checkpoints.py

@@ -129,7 +129,7 @@ def _read_checkpoint_loss(checkpoint_path: Path) -> Optional[float]:
     if not trainer_state.exists():
         return None
     try:
-        with open(trainer_state) as f:
+        with open(trainer_state, encoding = "utf-8") as f:
             state = json.load(f)
         log_history = state.get("log_history", [])
         if log_history:
@@ -174,18 +174,18 @@ def scan_checkpoints(
             metadata: dict = {}
             try:
                 if adapter_config.exists():
-                    cfg = json.loads(adapter_config.read_text())
+                    cfg = json.loads(adapter_config.read_text(encoding = "utf-8"))
                     metadata["base_model"] = cfg.get("base_model_name_or_path")
                     metadata["peft_type"] = cfg.get("peft_type")
                     metadata["lora_rank"] = cfg.get("r")
                 elif config_file.exists():
-                    cfg = json.loads(config_file.read_text())
+                    cfg = json.loads(config_file.read_text(encoding = "utf-8"))
                     metadata["base_model"] = cfg.get("_name_or_path")
 
                 # Detect BNB quantization from config.json
                 if config_file.exists():
                     if "cfg" not in dir():
-                        cfg = json.loads(config_file.read_text())
+                        cfg = json.loads(config_file.read_text(encoding = "utf-8"))
                     quant_cfg = cfg.get("quantization_config")
                     if (
                         isinstance(quant_cfg, dict)

studio/backend/utils/models/model_config.py

@@ -631,7 +631,7 @@ def _raw_config_has_vision_config(
                     cache_dir = active_hf_hub_cache(),
                 )
             )
-        config = json.loads(config_path.read_text())
+        config = json.loads(config_path.read_text(encoding = "utf-8"))
         architectures = config.get("architectures") or []
         model_type = config.get("model_type")
         explicit_vision = (
@@ -1083,7 +1083,7 @@ def _detect_audio_from_tokenizer(
                     ]:
                         tok_file = snapshot / tok_path
                         if tok_file.exists():
-                            tok_config = json.loads(tok_file.read_text())
+                            tok_config = json.loads(tok_file.read_text(encoding = "utf-8"))
                             read_any = True
                             result = _check_token_patterns(tok_config)
                             if result:
@@ -2283,7 +2283,7 @@ def scan_exported_models(
                 export_meta = run_dir / "export_metadata.json"
                 try:
                     if export_meta.exists():
-                        meta = json.loads(export_meta.read_text())
+                        meta = json.loads(export_meta.read_text(encoding = "utf-8"))
                         base_model = meta.get("base_model")
                 except Exception:
                     pass
@@ -2312,7 +2312,7 @@ def scan_exported_models(
                 if adapter_config.exists():
                     export_type = "lora"
                     try:
-                        cfg = json.loads(adapter_config.read_text())
+                        cfg = json.loads(adapter_config.read_text(encoding = "utf-8"))
                         base_model = cfg.get("base_model_name_or_path")
                     except Exception:
                         pass
@@ -2321,7 +2321,7 @@ def scan_exported_models(
                     export_meta = checkpoint_dir / "export_metadata.json"
                     try:
                         if export_meta.exists():
-                            meta = json.loads(export_meta.read_text())
+                            meta = json.loads(export_meta.read_text(encoding = "utf-8"))
                             base_model = meta.get("base_model")
                     except Exception:
                         pass
@@ -2334,7 +2334,7 @@ def scan_exported_models(
                         export_meta = meta_dir / "export_metadata.json"
                         try:
                             if export_meta.exists():
-                                meta = json.loads(export_meta.read_text())
+                                meta = json.loads(export_meta.read_text(encoding = "utf-8"))
                                 base_model = meta.get("base_model")
                                 if base_model:
                                     break
@@ -2354,7 +2354,7 @@ def scan_exported_models(
                     outputs_adapter_cfg = resolve_output_dir(run_dir.name) / "adapter_config.json"
                     try:
                         if outputs_adapter_cfg.exists():
-                            cfg = json.loads(outputs_adapter_cfg.read_text())
+                            cfg = json.loads(outputs_adapter_cfg.read_text(encoding = "utf-8"))
                             base_model = cfg.get("base_model_name_or_path")
                     except Exception:
                         pass
@@ -2380,7 +2380,7 @@ def get_base_model_from_checkpoint(checkpoint_path: str) -> Optional[str]:
 
         adapter_config_path = checkpoint_path_obj / "adapter_config.json"
         if adapter_config_path.exists():
-            with open(adapter_config_path, "r") as f:
+            with open(adapter_config_path, "r", encoding = "utf-8") as f:
                 config = json.load(f)
                 base_model = config.get("base_model_name_or_path")
                 if base_model:
@@ -2389,7 +2389,7 @@ def get_base_model_from_checkpoint(checkpoint_path: str) -> Optional[str]:
 
         config_path = checkpoint_path_obj / "config.json"
         if config_path.exists():
-            with open(config_path, "r") as f:
+            with open(config_path, "r", encoding = "utf-8") as f:
                 config = json.load(f)
                 for key in ("model_name", "_name_or_path"):
                     base_model = config.get(key)
@@ -2445,7 +2445,7 @@ def get_base_model_from_lora(lora_path: str) -> Optional[str]:
         # adapter_config.json first
         adapter_config_path = lora_path_obj / "adapter_config.json"
         if adapter_config_path.exists():
-            with open(adapter_config_path, "r") as f:
+            with open(adapter_config_path, "r", encoding = "utf-8") as f:
                 config = json.load(f)
                 base_model = config.get("base_model_name_or_path")
                 if base_model:
@@ -2535,7 +2535,7 @@ def get_base_model_from_lora_identifier(
             last_exc = exc
             continue
         try:
-            with open(cfg_path, "r") as f:
+            with open(cfg_path, "r", encoding = "utf-8") as f:
                 base_model = json.load(f).get("base_model_name_or_path")
         except Exception as exc:
             logger.warning("Could not parse adapter_config.json for '%s': %s", identifier, exc)
@@ -2781,7 +2781,7 @@ class ModelConfig:
                 meta_path = gguf_dir / "export_metadata.json"
                 if meta_path.exists():
                     try:
-                        meta = json.loads(meta_path.read_text())
+                        meta = json.loads(meta_path.read_text(encoding = "utf-8"))
                         base = meta.get("base_model")
                         if base and is_vision_model(base, hf_token = hf_token):
                             base_is_vision = True
@@ -2912,7 +2912,7 @@ class ModelConfig:
                         token = hf_token,
                         cache_dir = active_hf_hub_cache(),
                     )
-                    with open(config_path, "r") as f:
+                    with open(config_path, "r", encoding = "utf-8") as f:
                         adapter_config = json.load(f)
                     base_model = adapter_config.get("base_model_name_or_path")
                     if base_model:

studio/backend/utils/paths/path_utils.py

@@ -34,7 +34,7 @@ def _is_wsl() -> bool:
     if sys.platform == "win32":
         return False
     try:
-        with open("/proc/version", "r") as f:
+        with open("/proc/version", "r", encoding = "utf-8") as f:
             return "microsoft" in f.read().lower()
     except Exception:
         return False

studio/backend/utils/paths/storage_roots.py

@@ -212,7 +212,7 @@ def lmstudio_model_dirs() -> list[Path]:
     settings_path = Path.home() / ".lmstudio" / "settings.json"
     if settings_path.is_file():
         try:
-            with open(settings_path) as f:
+            with open(settings_path, encoding = "utf-8") as f:
                 settings = json.load(f)
             downloads = settings.get("downloadsFolder", "")
             if downloads:

studio/backend/utils/security/consent.py

@@ -142,7 +142,7 @@ def _load_remote_code_configs(model_name: str, hf_token: Optional[str] = None) -
             for name in _REMOTE_CODE_CONFIG_FILES:
                 p = root / name
                 if p.is_file():
-                    configs.append(json.loads(p.read_text()))
+                    configs.append(json.loads(p.read_text(encoding = "utf-8")))
             return configs
 
         from huggingface_hub import hf_hub_download
@@ -164,7 +164,7 @@ def _load_remote_code_configs(model_name: str, hf_token: Optional[str] = None) -
                 # Transient/auth failure is not "absent" -> fail closed to "unknown" so
                 # the caller scans (a tokenizer/processor-only auto_map must not slip by).
                 return None
-            configs.append(json.loads(Path(p).read_text()))
+            configs.append(json.loads(Path(p).read_text(encoding = "utf-8")))
         # Every config was read or a genuine 404 -> an empty list is a definitive
         # "no auto_map", not "unknown".
         return configs

studio/backend/utils/security/file_security.py

@@ -199,7 +199,9 @@ def _indexed_shard_paths(
                 inconclusive = True  # transient: an index that might exist could not be read
                 continue
             try:
-                weight_map = (json.loads(open(index_path).read()) or {}).get("weight_map") or {}
+                weight_map = (json.loads(open(index_path, encoding = "utf-8").read()) or {}).get(
+                    "weight_map"
+                ) or {}
                 for shard in weight_map.values():
                     shard_norm = _normalize_repo_path(str(shard))
                     # weight_map paths are relative to the index file's directory.
@@ -326,7 +328,7 @@ def _st_load_roots(snapshot: Path) -> list:
     roots = [snapshot]
     try:
         import json
-        modules = json.loads((snapshot / "modules.json").read_text())
+        modules = json.loads((snapshot / "modules.json").read_text(encoding = "utf-8"))
     except (OSError, ValueError):
         return roots  # no / invalid modules.json -> snapshot root is the only load root
     for module in modules or ():

studio/backend/utils/security/remote_code_approvals.py

@@ -69,7 +69,7 @@ def approval_target_key(targets) -> str:
 def _load() -> dict:
     """Parsed store, or an empty skeleton on any error (fail-safe = re-prompt)."""
     try:
-        with open(_store_path()) as f:
+        with open(_store_path(), encoding = "utf-8") as f:
             data = json.load(f)
         # Validate the shape, not just the version: a hand-edited ``subjects`` that is not a
         # dict (e.g. ``[]``) would otherwise crash lookup/record instead of failing safe.
@@ -92,7 +92,7 @@ def _save(data: dict) -> None:
     storage_roots.ensure_dir(path.parent)
     tmp = path.parent / f".{path.name}.tmp-{os.getpid()}"
     try:
-        with open(tmp, "w") as f:
+        with open(tmp, "w", encoding = "utf-8") as f:
             json.dump(data, f, indent = 2)
         try:
             os.chmod(tmp, 0o600)

studio/backend/utils/security/remote_code_scan.py

@@ -21,6 +21,8 @@ canonical scanner loads in-repo so the fallback never silently takes over.
 from __future__ import annotations
 
 import hashlib
+import io
+import tokenize
 import importlib.util
 import pathlib
 import re
@@ -392,6 +394,18 @@ def scan_remote_code_files(files: dict[str, str]) -> ScanResult:
     return result
 
 
+def _read_python_source(path) -> str:
+    """Decode a .py the way Python will execute it: a PEP 263 cookie
+    (`# coding: cp1252`) wins, so forcing utf-8 would scan something other than
+    what runs."""
+    data = path.read_bytes()
+    try:
+        encoding = tokenize.detect_encoding(io.BytesIO(data).readline)[0]
+    except (SyntaxError, ValueError):
+        encoding = "utf-8"
+    return data.decode(encoding, errors = "replace")
+
+
 def remote_code_fingerprint(files: dict[str, str]) -> str:
     """Stable sha256 over the (sorted) file contents, for pinning consent."""
     h = hashlib.sha256()
@@ -430,7 +444,7 @@ def repo_remote_code_files(model_name: str, hf_token: Optional[str] = None) -> d
             # for an RCE gate (HIGH stays approvable; only CRITICAL hard-blocks).
             for p in root.rglob("*.py"):
                 if p.is_file():
-                    files[str(p.relative_to(root))] = p.read_text(errors = "replace")
+                    files[str(p.relative_to(root))] = _read_python_source(p)
             # A local config can still point auto_map at an EXTERNAL Hub repo
             # (owner/name--module.Class) that executes on load, so fetch it. Every config
             # that can declare auto_map is checked, so a custom processor's external code
@@ -440,7 +454,7 @@ def repo_remote_code_files(model_name: str, hf_token: Optional[str] = None) -> d
                 p = root / name
                 if p.is_file():
                     try:
-                        ext_refs |= _auto_map_refs(json.loads(p.read_text()))
+                        ext_refs |= _auto_map_refs(json.loads(p.read_text(encoding = "utf-8")))
                     except Exception:
                         pass
             if not _add_external_refs(files, ext_refs, hf_token, model_name):
@@ -469,7 +483,7 @@ def repo_remote_code_files(model_name: str, hf_token: Optional[str] = None) -> d
                     f"{model_name}: config {cfg_name} could not be fetched ({exc})"
                 ) from exc
             try:
-                refs |= _auto_map_refs(json.loads(Path(cfg_path).read_text()))
+                refs |= _auto_map_refs(json.loads(Path(cfg_path).read_text(encoding = "utf-8")))
             except Exception:
                 pass
         own_refs = {fn for repo, fn in refs if repo is None}
@@ -519,7 +533,7 @@ def repo_remote_code_files(model_name: str, hf_token: Optional[str] = None) -> d
                 raise RemoteCodeUnscannable(
                     f"{model_name}: present file {fn} could not be fetched ({exc})"
                 ) from exc
-            files[fn] = Path(fp).read_text(errors = "replace")
+            files[fn] = _read_python_source(Path(fp))
         # Code referenced from another repo executes too: scan it or fail closed.
         if not _add_external_refs(files, refs, hf_token, model_name):
             raise RemoteCodeUnscannable(f"{model_name}: external auto_map code unreachable")
@@ -602,7 +616,7 @@ def external_auto_map_repos(model_name: str, hf_token: Optional[str] = None) ->
                 if not p.is_file():
                     continue
                 try:
-                    refs = _auto_map_refs(json.loads(p.read_text()))
+                    refs = _auto_map_refs(json.loads(p.read_text(encoding = "utf-8")))
                 except Exception:
                     continue
                 repos.update(repo for repo, _fn in refs if repo)
@@ -624,7 +638,7 @@ def external_auto_map_repos(model_name: str, hf_token: Optional[str] = None) ->
             except Exception:
                 continue
             try:
-                refs = _auto_map_refs(json.loads(Path(cfg_path).read_text()))
+                refs = _auto_map_refs(json.loads(Path(cfg_path).read_text(encoding = "utf-8")))
             except Exception:
                 continue
             repos.update(repo for repo, _fn in refs if repo)
@@ -701,5 +715,5 @@ def _add_external_refs(files: dict, refs, hf_token, model_name: str) -> bool:
                     exc,
                 )
                 return False
-            files[f"{repo}--{fn}"] = Path(fp).read_text(errors = "replace")
+            files[f"{repo}--{fn}"] = _read_python_source(Path(fp))
     return True

studio/backend/utils/transformers_version.py

@@ -420,7 +420,7 @@ def _resolve_base_model(model_name: str) -> str:
     adapter_cfg_path = local_path / "adapter_config.json"
     if _safe_is_file(adapter_cfg_path):
         try:
-            with open(adapter_cfg_path) as f:
+            with open(adapter_cfg_path, encoding = "utf-8") as f:
                 cfg = json.load(f)
             base = cfg.get("base_model_name_or_path")
             if base:
@@ -437,7 +437,7 @@ def _resolve_base_model(model_name: str) -> str:
     config_json_path = local_path / "config.json"
     if _safe_is_file(config_json_path):
         try:
-            with open(config_json_path) as f:
+            with open(config_json_path, encoding = "utf-8") as f:
                 cfg = json.load(f)
             # Unsloth writes model_name, HF writes _name_or_path; skip a self-reference.
             for _key in ("model_name", "_name_or_path"):
@@ -534,14 +534,19 @@ def _adapter_base_from_hf_cache(model_name: str) -> str | None:
     try:
         if ref_main.is_file():
             candidates.append(
-                repo_dir / "snapshots" / ref_main.read_text().strip() / "adapter_config.json"
+                repo_dir
+                / "snapshots"
+                / ref_main.read_text(encoding = "utf-8").strip()
+                / "adapter_config.json"
             )
         candidates += sorted(
             repo_dir.glob("snapshots/*/adapter_config.json"), key = _mtime, reverse = True
         )
         for cfg_path in candidates:
             if cfg_path.is_file():
-                base = json.loads(cfg_path.read_text()).get("base_model_name_or_path")
+                base = json.loads(cfg_path.read_text(encoding = "utf-8")).get(
+                    "base_model_name_or_path"
+                )
                 return base or None
     except Exception as exc:
         logger.debug("HF cache adapter_config.json lookup failed for '%s': %s", model_name, exc)
@@ -611,7 +616,7 @@ def _check_tokenizer_config_needs_v5(model_name: str, hf_token: str | None = Non
     local_tc = local_path / "tokenizer_config.json"
     if _safe_is_file(local_tc):
         try:
-            with open(local_tc) as f:
+            with open(local_tc, encoding = "utf-8") as f:
                 data = json.load(f)
             tokenizer_class = data.get("tokenizer_class", "")
             result = tokenizer_class in _TRANSFORMERS_5_TOKENIZER_CLASSES
@@ -688,7 +693,12 @@ def _config_json_from_hf_cache(model_name: str) -> dict | None:
     ref_main = repo_dir / "refs" / "main"
     try:
         if ref_main.is_file():
-            candidates.append(repo_dir / "snapshots" / ref_main.read_text().strip() / "config.json")
+            candidates.append(
+                repo_dir
+                / "snapshots"
+                / ref_main.read_text(encoding = "utf-8").strip()
+                / "config.json"
+            )
         # No refs/main (e.g. commit-pinned downloads): newest snapshot by mtime, not a stale
         # lexicographically-first SHA, matching what the Hub cache would actually load.
         candidates += sorted(
@@ -696,7 +706,7 @@ def _config_json_from_hf_cache(model_name: str) -> dict | None:
         )
         for cfg_path in candidates:
             if cfg_path.is_file():
-                with open(cfg_path) as f:
+                with open(cfg_path, encoding = "utf-8") as f:
                     return json.load(f)
     except Exception as exc:
         logger.debug("HF cache config.json lookup failed for '%s': %s", model_name, exc)
@@ -721,7 +731,7 @@ def _load_config_json(model_name: str, hf_token: str | None = None) -> dict | No
     local_cfg = Path(model_name) / "config.json"
     if _safe_is_file(local_cfg):
         try:
-            with open(local_cfg) as f:
+            with open(local_cfg, encoding = "utf-8") as f:
                 cfg = json.load(f)
             _config_json_cache[cache_key] = cfg
             return cfg
@@ -1755,7 +1765,7 @@ def _venv_dir_is_valid(venv_dir: str, packages: tuple[str, ...]) -> bool:
             metadata = di / "METADATA"
             if not metadata.is_file():
                 continue
-            for line in metadata.read_text(errors = "replace").splitlines():
+            for line in metadata.read_text(errors = "replace", encoding = "utf-8").splitlines():
                 if line.startswith("Version:"):
                     installed_ver = line.split(":", 1)[1].strip()
                     if installed_ver != pkg_version:
@@ -2391,7 +2401,10 @@ def _llmcompressor_shadow_is_valid() -> bool:
     """True if the shadow dir exists with a marker matching the current pin fingerprint."""
     marker = Path(_VENV_LLMCOMPRESSOR_DIR) / _LLMC_SHADOW_MARKER
     try:
-        return marker.is_file() and marker.read_text().strip() == _LLMC_SHADOW_FINGERPRINT
+        return (
+            marker.is_file()
+            and marker.read_text(encoding = "utf-8").strip() == _LLMC_SHADOW_FINGERPRINT
+        )
     except Exception:
         return False
 
@@ -2460,7 +2473,7 @@ def _ensure_venv_llmcompressor_exists() -> bool:
         if result.returncode == 0:
             try:
                 (Path(_VENV_LLMCOMPRESSOR_DIR) / _LLMC_SHADOW_MARKER).write_text(
-                    _LLMC_SHADOW_FINGERPRINT
+                    _LLMC_SHADOW_FINGERPRINT, encoding = "utf-8"
                 )
             except Exception:
                 pass

studio/backend/utils/utils.py

@@ -108,7 +108,7 @@ def hf_cache_snapshot_dir(model_name: str) -> Optional[Path]:
                 ref = repo_dir / "refs" / "main"
                 if not ref.is_file():
                     continue
-                commit = ref.read_text().strip()
+                commit = ref.read_text(encoding = "utf-8").strip()
                 if not commit:
                     continue
                 snapshot = repo_dir / "snapshots" / commit

studio/install_llama_prebuilt.py

@@ -2504,7 +2504,7 @@ def detect_host() -> HostInfo:
         if is_linux:
             for _vendor_file in glob.glob("/sys/class/drm/card*/device/vendor"):
                 try:
-                    with open(_vendor_file) as _vf:
+                    with open(_vendor_file, encoding = "utf-8") as _vf:
                         if _vf.read().strip().lower() == "0x8086":
                             has_intel_gpu = True
                             break
@@ -3050,7 +3050,7 @@ def _detect_host_rocm_version() -> tuple[int, int] | None:
         os.path.join(rocm_root, "lib", "rocm_version"),
     ):
         try:
-            with open(path) as fh:
+            with open(path, encoding = "utf-8") as fh:
                 parts = fh.read().strip().split("-")[0].split(".")
             # Explicit length guard avoids relying on the broad except
             # below to swallow IndexError when the version file contains
@@ -5526,7 +5526,9 @@ def write_prebuilt_metadata(
         "prebuilt_fallback_used": prebuilt_fallback_used,
         "installed_at_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
     }
-    (install_dir / "UNSLOTH_PREBUILT_INFO.json").write_text(json.dumps(metadata, indent = 2) + "\n")
+    (install_dir / "UNSLOTH_PREBUILT_INFO.json").write_text(
+        json.dumps(metadata, indent = 2) + "\n", encoding = "utf-8"
+    )
 
 
 def sync_marker_force_cpu(install_dir: Path, persist_force_cpu: bool) -> None:
@@ -5537,13 +5539,13 @@ def sync_marker_force_cpu(install_dir: Path, persist_force_cpu: bool) -> None:
     GPU/Vulkan bundle that revives the crash (#7213)."""
     marker_path = install_dir / "UNSLOTH_PREBUILT_INFO.json"
     try:
-        marker = json.loads(marker_path.read_text())
+        marker = json.loads(marker_path.read_text(encoding = "utf-8"))
     except (OSError, ValueError):
         return
     if not isinstance(marker, dict) or bool(marker.get("force_cpu")) == persist_force_cpu:
         return
     marker["force_cpu"] = persist_force_cpu
-    marker_path.write_text(json.dumps(marker, indent = 2) + "\n")
+    marker_path.write_text(json.dumps(marker, indent = 2) + "\n", encoding = "utf-8")
     log(f"existing install reused; recorded force_cpu={persist_force_cpu} from this run")
 
 

studio/install_node_prebuilt.py

@@ -535,7 +535,7 @@ def install_lock(lock_path: Path) -> Iterator[None]:
                 break
             except FileExistsError:
                 try:
-                    raw = lock_path.read_text().strip()
+                    raw = lock_path.read_text(encoding = "utf-8").strip()
                 except FileNotFoundError:
                     continue
                 stale = False
@@ -660,7 +660,7 @@ def write_metadata(install_dir: Path, *, version: str, asset: str, sha256: str)
         "asset": asset,
         "sha256": sha256,
     }
-    metadata_path(install_dir).write_text(json.dumps(payload, indent = 2) + "\n")
+    metadata_path(install_dir).write_text(json.dumps(payload, indent = 2) + "\n", encoding = "utf-8")
 
 
 def load_metadata(install_dir: Path) -> dict | None:
@@ -668,7 +668,7 @@ def load_metadata(install_dir: Path) -> dict | None:
     if not path.exists():
         return None
     try:
-        data = json.loads(path.read_text())
+        data = json.loads(path.read_text(encoding = "utf-8"))
     except (json.JSONDecodeError, OSError):
         return None
     return data if isinstance(data, dict) else None

studio/install_python_stack.py

@@ -503,7 +503,7 @@ def _detect_rocm_version() -> tuple[int, int] | None:
         os.path.join(rocm_root, "lib", "rocm_version"),
     ):
         try:
-            with open(path) as fh:
+            with open(path, encoding = "utf-8") as fh:
                 parts = fh.read().strip().split("-")[0].split(".")
             # Explicit length guard: don't rely on the broad except below to
             # swallow IndexError on a single-component version (e.g. "6\n").
@@ -852,9 +852,9 @@ def _linux_amd_display_device_present() -> bool:
     try:
         for dev in Path("/sys/bus/pci/devices").iterdir():
             try:
-                if (dev / "vendor").read_text().strip() != "0x1002":
+                if (dev / "vendor").read_text(encoding = "utf-8").strip() != "0x1002":
                     continue
-                if (dev / "class").read_text().strip().startswith("0x03"):
+                if (dev / "class").read_text(encoding = "utf-8").strip().startswith("0x03"):
                     return True
             except OSError:
                 continue
@@ -1067,7 +1067,7 @@ def _has_rocm_gpu() -> bool:
                 for entry in os.listdir(kfd_nodes):
                     gpu_id_path = os.path.join(kfd_nodes, entry, "gpu_id")
                     try:
-                        with open(gpu_id_path) as fh:
+                        with open(gpu_id_path, encoding = "utf-8") as fh:
                             gpu_id = fh.read().strip()
                     except OSError:
                         continue
@@ -1079,7 +1079,7 @@ def _has_rocm_gpu() -> bool:
                     # false positive (e.g. NVIDIA open-driver KFD nodes lacking it).
                     props_path = os.path.join(kfd_nodes, entry, "properties")
                     try:
-                        with open(props_path) as fh:
+                        with open(props_path, encoding = "utf-8") as fh:
                             props = fh.read()
                     except OSError:
                         continue  # can't confirm vendor -- skip

studio/prebuilt_core.py

@@ -1078,7 +1078,7 @@ def install_lock(lock_path: Path) -> Iterator[None]:
             except FileExistsError:
                 stale = False
                 try:
-                    raw = lock_path.read_text().strip()
+                    raw = lock_path.read_text(encoding = "utf-8").strip()
                 except FileNotFoundError:
                     # Lock vanished between our open and read -- retry
                     continue

tests/test_runtime_text_encoding.py

@@ -0,0 +1,407 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Guard: shipping code must name an encoding on every text read and write.
+
+`Path.read_text()`, `Path.write_text()`, `Path.open()` and builtin `open()` fall back
+to `locale.getencoding()`: UTF-8 on the Linux and macOS runners, cp1252 on a stock
+Windows install. Every file this repo reads at runtime is UTF-8 (HF `config.json` /
+`tokenizer_config.json` / `adapter_config.json`, Ollama manifests, GGUF export
+metadata), so on Windows those reads crash or, worse, succeed with mojibake: a
+DeepSeek or Qwen tokenizer_config.json carries U+FF5C and U+2581 in its chat
+template, and at utils/models/model_config.py that read sits inside a broad
+`except Exception: logger.debug(...)`, so the token-pattern check silently
+returned the wrong answer.
+
+Unlike the import-time rule in test_source_read_encoding.py this is scope agnostic:
+runtime reads live inside functions, and shipping code has no legitimate reason to
+let the operator's locale decide. No reachability analysis to get wrong, so no
+allowlist and no false positives.
+
+Binary handles are skipped (no encoding to name, and passing one is a ValueError),
+and a non-constant mode counts as unknown rather than text: demanding `encoding =`
+on a call that may resolve to "rb" would leave no compliant way to write it.
+
+Known limitation, deliberately not closed: `configparser.ConfigParser.read()` also
+defaults to the locale encoding, but cannot be matched by name without resolving the
+receiver, since `f.read(n)`, `resp.read(limit)` and `handle.read(chunk)` are spelled
+identically. Flagging it would be a false positive with no compliant fix, the exact
+failure mode this guard avoids. The one live `ConfigParser.read` (/etc/wsl.conf,
+hub/utils/paths.py) is pinned by hand; a future one has to be caught in review.
+"""
+
+# `str | None` below is evaluated at import on Python 3.9 (requires-python >= 3.9).
+from __future__ import annotations
+
+import ast
+import subprocess
+from pathlib import Path
+
+
+REPO = Path(__file__).resolve().parent.parent
+# Everything that ships. `studio/` covers the installers too: install_python_stack.py
+# reads /sys/class/kfd, the same detection path as utils/hardware/hardware.py. Test
+# trees fall under the narrower import-time rule in test_source_read_encoding.py.
+ROOTS = (REPO / "unsloth", REPO / "studio", REPO / "unsloth_cli")
+# The frontend tree is TypeScript; node_modules is vendored third-party code.
+SKIP_DIRS = {"build", "dist", "frontend", "node_modules", "src-tauri", ".venv", "site-packages"}
+GUARDED_METHODS = {"read_text", "write_text"}
+# Path classes, so an unbound `Path.open(p)` shifts every argument one right.
+PATH_CLASSES = {"Path", "PosixPath", "PurePath", "WindowsPath"}
+# Values that re-select the platform default when passed as the encoding.
+PLATFORM_DEFAULT_ENCODINGS = (None, "locale")
+# Calls that return the platform default, so naming one pins nothing.
+PLATFORM_DEFAULT_CALLS = {"getdefaultencoding", "getencoding", "getpreferredencoding"}
+# Modules whose `open` IS the builtin: same signature, same platform default.
+BUILTIN_OPEN_MODULES = {"builtins", "io"}
+# Take an encoding in "t" mode but default to "rb". Value is its positional slot.
+COMPRESSED_OPENERS = {"bz2": 3, "gzip": 3, "lzma": None}
+# Distinct from None so that "no mode argument at all" still means text.
+UNKNOWN_MODE = object()
+
+
+def _mode(call: ast.Call, positional_index: int):
+    """The call's mode, or UNKNOWN_MODE when it is not a literal."""
+    # A splat hides the mode, so it is unknown rather than absent: falling through to
+    # "r" would flag a call that may resolve to binary, with no compliant way to fix it.
+    if any(isinstance(a, ast.Starred) for a in call.args):
+        return UNKNOWN_MODE
+    if any(kw.arg is None for kw in call.keywords):
+        return UNKNOWN_MODE
+    if len(call.args) > positional_index:
+        node = call.args[positional_index]
+        return node.value if isinstance(node, ast.Constant) else UNKNOWN_MODE
+    for kw in call.keywords:
+        if kw.arg == "mode":
+            return kw.value.value if isinstance(kw.value, ast.Constant) else UNKNOWN_MODE
+    return "r"
+
+
+def _names_encoding(call: ast.Call) -> bool:
+    """True only for an encoding that actually pins one.
+
+    `encoding = None` and `encoding = "locale"` re-select the platform default, so the
+    keyword being present is not enough. A `**kwargs` splat may carry an encoding we
+    cannot see, so it counts as named rather than as an unsatisfiable demand.
+    """
+    for kw in call.keywords:
+        if kw.arg is None:
+            return True
+        if kw.arg != "encoding":
+            continue
+        if isinstance(kw.value, ast.Constant) and kw.value.value in PLATFORM_DEFAULT_ENCODINGS:
+            return False
+        if isinstance(kw.value, ast.Call) and _callee_name(kw.value.func) in PLATFORM_DEFAULT_CALLS:
+            return False  # locale.getencoding() is the default, spelled out
+        return True
+    return False
+
+
+def _is_text(call: ast.Call, positional_index: int) -> bool:
+    mode = _mode(call, positional_index)
+    return mode is not UNKNOWN_MODE and "b" not in str(mode)
+
+
+def _imports_at_each_call(tree: ast.Module) -> dict:
+    """The imports visible at every call, keyed by node id.
+
+    A function's own imports stay in that function: hoisting them would let one local
+    `from PIL.Image import open` turn off the builtin check for the whole file.
+    """
+    visible_at = {}
+
+    def walk(node, visible):
+        if isinstance(node, ast.Call):
+            visible_at[id(node)] = visible
+        for child in ast.iter_child_nodes(node):
+            if isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)):
+                walk(child, {**visible, **_imported_names(child)})
+            else:
+                walk(child, visible)
+
+    walk(tree, _imported_names(tree))
+    return visible_at
+
+
+def _foreign_names(tree: ast.Module) -> set:
+    """Names bound to an object another library built.
+
+    `z = zipfile.ZipFile(p)` then `z.open(name)` is a binary member stream taking no
+    encoding, so demanding one leaves no correct edit.
+    """
+    modules = _imported_names(tree)
+    names = set()
+    for node in ast.walk(tree):
+        if not isinstance(node, ast.Assign) or not isinstance(node.value, ast.Call):
+            continue
+        if _foreign_receiver(node.value, modules):
+            names.update(t.id for t in node.targets if isinstance(t, ast.Name))
+    return names
+
+
+def _imported_names(tree) -> dict:
+    """Names this module's imports bind, mapped to where they came from.
+
+    The name alone settles nothing: `import tarfile as tf` hides an opener that takes
+    no encoding, and `from PIL.Image import open` puts another behind the most familiar
+    name there is. Resolving the origin covers both, with no module list to maintain.
+    """
+    bound = {}
+    stack = list(ast.iter_child_nodes(tree))
+    while stack:
+        node = stack.pop()
+        if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)):
+            continue  # that function's business, not this scope's
+        if isinstance(node, ast.Import):
+            for a in node.names:
+                bound[(a.asname or a.name).split(".")[0]] = a.name
+        elif isinstance(node, ast.ImportFrom):
+            for a in node.names:
+                bound[a.asname or a.name] = f"{node.module}.{a.name}" if node.module else a.name
+        else:
+            stack.extend(ast.iter_child_nodes(node))
+    return bound
+
+
+def _callee_name(func):
+    """The bare name a callee ends in, whether or not it is qualified."""
+    return func.id if isinstance(func, ast.Name) else getattr(func, "attr", None)
+
+
+def _origin_root(name, modules) -> str:
+    """The top-level module a bound name came from, or the name itself."""
+    return modules.get(name, name).split(".")[0]
+
+
+def _compressed_key(name, modules):
+    """The COMPRESSED_OPENERS entry this receiver resolves to, if any."""
+    for candidate in (name, _origin_root(name, modules)):
+        if candidate in COMPRESSED_OPENERS:
+            return candidate
+    return None
+
+
+def _open_alias(name, modules):
+    """What a bare callable resolves to: "builtin", a COMPRESSED_OPENERS key, or None."""
+    origin = modules.get(name)
+    if origin is None:
+        return "builtin" if name == "open" else None
+    parts = origin.split(".")
+    if parts[-1] != "open":
+        return None
+    if parts[0] in BUILTIN_OPEN_MODULES or origin == "open":
+        return "builtin"
+    return parts[0] if parts[0] in COMPRESSED_OPENERS else None
+
+
+def _is_path_class(name, modules) -> bool:
+    """True for a pathlib class, including under an alias."""
+    if name is None:
+        return False
+    return (modules.get(name) or name).split(".")[-1] in PATH_CLASSES
+
+
+def _is_path_attr(node) -> bool:
+    """True for a qualified path class, as in `pathlib.Path`."""
+    return isinstance(node, ast.Attribute) and node.attr in PATH_CLASSES
+
+
+def _foreign_receiver(node, modules) -> bool:
+    """True when the thing before `.open` is an object another library built.
+
+    `zipfile.ZipFile(p).open(name)` returns a binary member stream taking no encoding,
+    so it needs the same exemption as the bare `zipfile.open` spelling.
+    """
+    if not isinstance(node, ast.Call):
+        return False
+    func = node.func
+    if isinstance(func, ast.Attribute) and isinstance(func.value, ast.Name):
+        root, name = func.value.id, func.attr
+    elif isinstance(func, ast.Name):
+        root = name = func.id
+    else:
+        return False
+    return root in modules and not _is_path_class(name, modules)
+
+
+def _offender(
+    call: ast.Call,
+    modules = None,
+    foreign = (),
+) -> str | None:
+    """The call's name if it does text I/O without pinning an encoding."""
+    modules = {} if modules is None else modules
+    func = call.func
+    if isinstance(func, ast.Attribute):
+        receiver = func.value.id if isinstance(func.value, ast.Name) else None
+        # `Path.read_text(p)` is `p.read_text()` unbound: the instance takes slot 0,
+        # so every argument shifts one place right.
+        shift = 1 if _is_path_class(receiver, modules) or _is_path_attr(func.value) else 0
+        if func.attr in GUARDED_METHODS:
+            if func.attr == "read_text" and not shift and call.args:
+                first = call.args[0]
+                # Bound read_text takes encoding first, so None or "locale" there is a
+                # platform-default read. Any other positional means the receiver is
+                # importlib.metadata's Distribution: a filename, and no encoding at all.
+                if isinstance(first, ast.Constant) and first.value in PLATFORM_DEFAULT_ENCODINGS:
+                    return "read_text()"
+                return None
+            return None if _names_encoding(call) else f"{func.attr}()"
+        if func.attr == "open":
+            if receiver is not None and _origin_root(receiver, modules) in BUILTIN_OPEN_MODULES:
+                return (
+                    None if not _is_text(call, 1) or _names_encoding(call) else f"{receiver}.open()"
+                )
+            compressed = _compressed_key(receiver, modules) if receiver else None
+            if compressed is not None:
+                # "rb" by default, so only an explicit text mode is in scope.
+                mode = _mode(call, 1)
+                if mode is UNKNOWN_MODE or "t" not in str(mode):
+                    return None
+                return None if _names_encoding(call) else f"{compressed}.open()"
+            # Any other imported receiver is somebody else's opener: tarfile takes a
+            # compression mode, Image a binary file. Neither has an encoding to name.
+            if receiver is not None and receiver in modules and receiver not in PATH_CLASSES:
+                return None
+            if _foreign_receiver(func.value, modules) or receiver in foreign:
+                return None
+            if not _is_text(call, shift):
+                return None
+            return None if _names_encoding(call) else "Path.open()"
+        return None
+    if isinstance(func, ast.Name):
+        alias = _open_alias(func.id, modules)
+        if alias == "builtin":
+            if not _is_text(call, 1):
+                return None
+            return None if _names_encoding(call) else "open()"
+        if alias is not None:
+            mode = _mode(call, 1)
+            if mode is UNKNOWN_MODE or "t" not in str(mode):
+                return None
+            return None if _names_encoding(call) else f"{alias}.open()"
+    return None
+
+
+def _is_test_path(path: Path) -> bool:
+    parts = path.relative_to(REPO).parts
+    if SKIP_DIRS.intersection(parts):
+        return True
+    if "tests" in parts or "test" in parts:
+        return True
+    return path.name.startswith("test_") or path.name.endswith("_test.py")
+
+
+def _offenders_in(src: str, label: str = "<snippet>"):
+    tree = ast.parse(src, filename = label)
+    visible_at = _imports_at_each_call(tree)
+    foreign = _foreign_names(tree)
+    found = []
+    for node in ast.walk(tree):
+        if isinstance(node, ast.Call):
+            name = _offender(node, visible_at.get(id(node), {}), foreign)
+            if name is not None:
+                found.append((node.lineno, name))
+    return found
+
+
+def _tracked_sources():
+    """Shipping *.py that git is actually tracking.
+
+    A walk also picks up whatever is lying in the checkout (a built `build/lib` copy,
+    a nested worktree, a vendored dep). None of those are ours to police, and a stale
+    artifact would fail this for everybody who has one.
+    """
+    listed = subprocess.run(
+        ["git", "-C", str(REPO), "ls-files", "-z", "--", "*.py"],
+        capture_output = True,
+        timeout = 60,
+    )
+    if listed.returncode != 0:
+        return None  # not a checkout, so fall back to walking
+    names = listed.stdout.decode("utf-8", errors = "replace").split("\0")
+    return [REPO / n for n in names if n]
+
+
+def _walked_sources():
+    return [p for root in ROOTS if root.is_dir() for p in sorted(root.rglob("*.py"))]
+
+
+def test_shipping_code_names_an_encoding():
+    offenders = []
+    sources = _tracked_sources()
+    if sources is None:
+        sources = _walked_sources()
+    roots = {r.resolve() for r in ROOTS}
+    for path in sorted(sources):
+        if not roots.intersection(path.resolve().parents) or _is_test_path(path):
+            continue
+        try:
+            tree = ast.parse(path.read_text(encoding = "utf-8"), filename = str(path))
+        except SyntaxError:
+            continue
+        rel = path.relative_to(REPO).as_posix()
+        visible_at = _imports_at_each_call(tree)
+        foreign = _foreign_names(tree)
+        for node in ast.walk(tree):
+            if isinstance(node, ast.Call):
+                name = _offender(node, visible_at.get(id(node), {}), foreign)
+                if name is not None:
+                    offenders.append(f"{rel}:{node.lineno}: {name}")
+    assert offenders == [], (
+        f"{len(offenders)} text read/write call sites in shipping code let the "
+        "operator's locale decide the encoding, so they crash or silently "
+        'produce mojibake on Windows. Pass encoding = "utf-8": ' + repr(offenders)
+    )
+
+
+# The assertion above passes vacuously once the trees are clean, so it cannot tell a
+# working detector from one that always returns None. These pin the detector itself.
+
+
+def test_detects_the_plain_cases():
+    assert _offenders_in("from pathlib import Path\np = Path('x')\ns = p.read_text()\n")
+    assert _offenders_in("p.write_text('hi')\n")
+    assert _offenders_in("f = open('x')\n")
+    assert _offenders_in("f = open('x', 'w')\n")
+    assert _offenders_in("f = p.open()\n")
+    # Inside a function body too: shipping reads are not import-time.
+    assert _offenders_in("def load(p):\n    return p.read_text()\n")
+
+
+def test_rejects_encoding_that_reselects_the_platform_default():
+    assert _offenders_in("s = p.read_text(encoding = None)\n")
+    assert _offenders_in("s = p.read_text(encoding = 'locale')\n")
+
+
+def test_accepts_a_pinned_encoding():
+    assert not _offenders_in("s = p.read_text(encoding = 'utf-8')\n")
+    assert not _offenders_in("f = open('x', 'w', encoding = 'utf-8')\n")
+    assert not _offenders_in("f = p.open(encoding = 'utf-8')\n")
+    assert not _offenders_in("s = p.read_text(encoding = 'utf-8', errors = 'replace')\n")
+
+
+def test_skips_binary_handles():
+    # Binary has no encoding to name; passing one is a ValueError.
+    assert not _offenders_in("f = open('x', 'rb')\n")
+    assert not _offenders_in("f = open('x', mode = 'wb')\n")
+    assert not _offenders_in("f = p.open('rb')\n")
+
+
+def test_skips_unknown_modes():
+    # A call that may resolve to "rb" has no compliant way to name an encoding.
+    assert not _offenders_in("mode = 'rb' if binary else 'r'\nf = open(path, mode)\n")
+    assert not _offenders_in("f = open(path, mode = chosen)\n")
+
+
+def test_skips_foreign_openers_and_readers():
+    assert not _offenders_in("import fitz\nd = fitz.open(stream = b, filetype = 'pdf')\n")
+    assert not _offenders_in("import tarfile\nt = tarfile.open(p, 'r:gz')\n")
+    # importlib.metadata Distribution.read_text takes a positional filename.
+    assert not _offenders_in("s = dist.read_text('direct_url.json')\n")
+
+
+def test_test_trees_are_out_of_scope():
+    assert _is_test_path(REPO / "tests" / "test_x.py")
+    assert _is_test_path(REPO / "studio" / "backend" / "tests" / "helpers.py")
+    assert not _is_test_path(REPO / "studio" / "backend" / "routes" / "inference.py")

unsloth/models/_utils.py

@@ -2471,7 +2471,7 @@ def _get_statistics(statistics = None, force_download = True):
                     for vendor_file in vendor_files:
                         path = Path(vendor_file)
                         if path.is_file():
-                            file_content = path.read_text().lower()
+                            file_content = path.read_text(encoding = "utf-8").lower()
                             if "amazon" in file_content:
                                 return "aws"
                             elif "microsoft corporation" in file_content:

unsloth/models/loader.py

@@ -1585,7 +1585,7 @@ class FastModel(FastBaseModel):
         if do_logging:
             redirector = contextlib.nullcontext()
         else:
-            redirector = contextlib.redirect_stdout(open(os.devnull, "w"))
+            redirector = contextlib.redirect_stdout(open(os.devnull, "w", encoding = "utf-8"))
 
         model_types = ["siglip"] + model_types
         # Set forced float32 env flag

unsloth/models/loader_utils.py

@@ -450,7 +450,7 @@ def _load_fp8_weight_map(
         index_path = None
     if index_path is not None:
         import json
-        with open(index_path, "r") as f:
+        with open(index_path, "r", encoding = "utf-8") as f:
             return json.load(f).get("weight_map", None)
 
     # Unsharded single file: map every tensor to it.

unsloth/models/sentence_transformer.py

@@ -2329,7 +2329,7 @@ def _patch_st_trainer_load_from_checkpoint():
         if not os.path.isfile(modules_json):
             raise RuntimeError("Unsloth: PEFT checkpoint is missing modules.json.")
         try:
-            with open(modules_json, "r") as f:
+            with open(modules_json, "r", encoding = "utf-8") as f:
                 module_configs = json.load(f)
         except Exception as e:
             raise RuntimeError("Unsloth: Cannot parse checkpoint modules.json.") from e

unsloth_cli/_inference.py

@@ -98,7 +98,7 @@ def _json_rank_count_from_env(name: str) -> Optional[int]:
         if value.lstrip().startswith(("[", "{")):
             data = json.loads(value)
         else:
-            with open(value, "r") as f:
+            with open(value, "r", encoding = "utf-8") as f:
                 data = json.load(f)
     except (OSError, json.JSONDecodeError):
         return None
@@ -158,7 +158,7 @@ def quiet_if_nonzero_mlx_rank():
     sys.stderr.flush()
     saved_stdout_fd = os.dup(1)
     saved_stderr_fd = os.dup(2)
-    with open(os.devnull, "w") as devnull:
+    with open(os.devnull, "w", encoding = "utf-8") as devnull:
         try:
             os.dup2(devnull.fileno(), 1)
             os.dup2(devnull.fileno(), 2)

unsloth_cli/commands/studio.py

@@ -719,7 +719,7 @@ def _cli_update_password(conn: sqlite3.Connection, username: str, new_password:
             # credential after a later reset-password deletes auth.db. Mirrors
             # backend clear_bootstrap_password().
             try:
-                stale_path.write_text("")
+                stale_path.write_text("", encoding = "utf-8")
                 cleared = True
             except OSError:
                 cleared = False
@@ -2406,7 +2406,7 @@ def stop():
         typer.echo("No running Unsloth server found (no PID file).")
         raise typer.Exit(0)
 
-    pid_text = _PID_FILE.read_text().strip()
+    pid_text = _PID_FILE.read_text(encoding = "utf-8").strip()
     if not pid_text.isdigit():
         typer.echo(f"Invalid PID file contents: {pid_text}")
         _PID_FILE.unlink(missing_ok = True)
@@ -2863,7 +2863,7 @@ def reset_password():
             path.unlink(missing_ok = True)
         except OSError:
             try:
-                path.write_text("")
+                path.write_text("", encoding = "utf-8")
             except OSError as exc:
                 typer.echo(
                     f"Error: could not remove or clear {path.name} ({exc}); delete "