From 3fd948eb952e417c7604a9422a55f7fb130a72cf Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 27 Jul 2026 02:14:20 -0700 Subject: [PATCH] Pin utf-8 on shipping-code text I/O instead of the operator locale (#7486) * Pin utf-8 on shipping-code text I/O instead of the operator locale 113 read_text/write_text/open call sites across unsloth, studio and unsloth_cli let locale.getencoding() decide the encoding. That is utf-8 on the Linux and macOS runners and cp1252 on a stock Windows install, so the same file decodes differently for a Windows user and silently produces mojibake or raises UnicodeDecodeError. Adds tests/test_runtime_text_encoding.py to keep it that way. It resolves openers through each file's own imports rather than a fixed list of module names, so an aliased tarfile.open or a local from PIL.Image import open is not asked for an encoding it does not take. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Scan tracked files only and resolve the unbound Path calling forms * Honour PEP 263 when scanning sources and migrate a legacy JSONL before appending * Scope guard imports lexically and only migrate a legacy file when it round-trips * Leave a legacy JSONL untouched and resolve path aliases in the foreign-opener check * Tighten comments --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- studio/backend/auth/storage.py | 8 +- studio/backend/colab.py | 4 +- studio/backend/core/export/export.py | 4 +- studio/backend/core/inference/inference.py | 2 +- studio/backend/core/inference/llama_cpp.py | 12 +- .../inference/sandbox_site/sitecustomize.py | 4 +- studio/backend/core/inference/worker.py | 7 +- studio/backend/core/rag/embeddings.py | 4 +- studio/backend/hub/services/models/ollama.py | 4 +- studio/backend/hub/utils/download_registry.py | 4 +- studio/backend/hub/utils/paths.py | 4 +- studio/backend/main.py | 6 +- .../scraper_impl/state_store.py | 10 +- .../data_designer_unstructured_seed/impl.py | 2 +- studio/backend/routes/inference.py | 2 +- studio/backend/routes/models.py | 6 +- studio/backend/run.py | 4 +- studio/backend/utils/hardware/hardware.py | 18 +- studio/backend/utils/models/checkpoints.py | 8 +- studio/backend/utils/models/model_config.py | 26 +- studio/backend/utils/paths/path_utils.py | 2 +- studio/backend/utils/paths/storage_roots.py | 2 +- studio/backend/utils/security/consent.py | 4 +- .../backend/utils/security/file_security.py | 6 +- .../utils/security/remote_code_approvals.py | 4 +- .../utils/security/remote_code_scan.py | 28 +- studio/backend/utils/transformers_version.py | 35 +- studio/backend/utils/utils.py | 2 +- studio/install_llama_prebuilt.py | 12 +- studio/install_node_prebuilt.py | 6 +- studio/install_python_stack.py | 10 +- studio/prebuilt_core.py | 2 +- tests/test_runtime_text_encoding.py | 407 ++++++++++++++++++ unsloth/models/_utils.py | 2 +- unsloth/models/loader.py | 2 +- unsloth/models/loader_utils.py | 2 +- unsloth/models/sentence_transformer.py | 2 +- unsloth_cli/_inference.py | 4 +- unsloth_cli/commands/studio.py | 6 +- 39 files changed, 563 insertions(+), 114 deletions(-) create mode 100644 tests/test_runtime_text_encoding.py diff --git a/studio/backend/auth/storage.py b/studio/backend/auth/storage.py index 39fa691304..5f80ad89a3 100644 --- a/studio/backend/auth/storage.py +++ b/studio/backend/auth/storage.py @@ -44,7 +44,7 @@ def generate_bootstrap_password() -> str: # Persisted from a previous run? if _BOOTSTRAP_PW_PATH.is_file(): - _bootstrap_password = _BOOTSTRAP_PW_PATH.read_text().strip() + _bootstrap_password = _BOOTSTRAP_PW_PATH.read_text(encoding = "utf-8").strip() if _bootstrap_password: return _bootstrap_password @@ -57,7 +57,7 @@ def generate_bootstrap_password() -> str: # Persist so the same passphrase survives restarts until password change. ensure_dir(_BOOTSTRAP_PW_PATH.parent) - _BOOTSTRAP_PW_PATH.write_text(_bootstrap_password) + _BOOTSTRAP_PW_PATH.write_text(_bootstrap_password, encoding = "utf-8") try: os.chmod(_BOOTSTRAP_PW_PATH, 0o600) except OSError: @@ -76,7 +76,7 @@ def _load_bootstrap_password() -> Optional[str]: global _bootstrap_password _bootstrap_password = None if _BOOTSTRAP_PW_PATH.is_file(): - bootstrap_password = _BOOTSTRAP_PW_PATH.read_text().strip() + bootstrap_password = _BOOTSTRAP_PW_PATH.read_text(encoding = "utf-8").strip() if bootstrap_password: _bootstrap_password = bootstrap_password return _bootstrap_password @@ -99,7 +99,7 @@ def clear_bootstrap_password() -> None: # stale plaintext can't be re-seeded by generate_bootstrap_password() # if a later reset-password deletes auth.db and re-validates it. try: - _BOOTSTRAP_PW_PATH.write_text("") + _BOOTSTRAP_PW_PATH.write_text("", encoding = "utf-8") cleared = True except OSError: cleared = False diff --git a/studio/backend/colab.py b/studio/backend/colab.py index 051d80abfe..df1285b749 100644 --- a/studio/backend/colab.py +++ b/studio/backend/colab.py @@ -90,7 +90,7 @@ def _store_colab_login_credentials(username: str, password: str) -> None: path = _colab_login_credentials_path() try: path.parent.mkdir(parents = True, exist_ok = True) - path.write_text(f"{username}\n{password}\n") + path.write_text(f"{username}\n{password}\n", encoding = "utf-8") try: import os os.chmod(path, 0o600) @@ -106,7 +106,7 @@ def _load_colab_login_credentials() -> "tuple[str, str] | None": try: if not path.is_file(): return None - lines = path.read_text().splitlines() + lines = path.read_text(encoding = "utf-8").splitlines() if len(lines) >= 2 and lines[0] and lines[1]: return lines[0], lines[1] except OSError as e: diff --git a/studio/backend/core/export/export.py b/studio/backend/core/export/export.py index e364ea4f3a..4979ebd48d 100644 --- a/studio/backend/core/export/export.py +++ b/studio/backend/core/export/export.py @@ -241,7 +241,7 @@ def _offline_window_if(local_files_only): def _is_wsl(): """Detect if running under Windows Subsystem for Linux.""" try: - return "microsoft" in open("/proc/version").read().lower() + return "microsoft" in open("/proc/version", encoding = "utf-8").read().lower() except Exception: return False @@ -574,7 +574,7 @@ class ExportBackend: ) metadata = {"base_model": base_model} metadata_path = os.path.join(save_directory, "export_metadata.json") - with open(metadata_path, "w") as f: + with open(metadata_path, "w", encoding = "utf-8") as f: json.dump(metadata, f, indent = 2) logger.info(f"Wrote export metadata to {metadata_path}") except Exception as e: diff --git a/studio/backend/core/inference/inference.py b/studio/backend/core/inference/inference.py index 2f46470091..563a6732a1 100644 --- a/studio/backend/core/inference/inference.py +++ b/studio/backend/core/inference/inference.py @@ -567,7 +567,7 @@ class InferenceBackend: _meta_path = Path(config.path) / "export_metadata.json" try: if _meta_path.exists(): - _meta = json.loads(_meta_path.read_text()) + _meta = json.loads(_meta_path.read_text(encoding = "utf-8")) if _meta.get("base_model"): processor_source = _meta["base_model"] except Exception: diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 4035188e88..f286a2e4c5 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -569,7 +569,7 @@ def _load_swa_cache() -> dict: if _SWA_CACHE is not None: return _SWA_CACHE try: - with open(_swa_cache_path()) as f: + with open(_swa_cache_path(), encoding = "utf-8") as f: _SWA_CACHE = json.load(f) if not isinstance(_SWA_CACHE, dict): _SWA_CACHE = {} @@ -583,7 +583,7 @@ def _save_swa_cache(cache: dict) -> None: path = _swa_cache_path() path.parent.mkdir(parents = True, exist_ok = True) tmp = path.with_suffix(".json.tmp") - with open(tmp, "w") as f: + with open(tmp, "w", encoding = "utf-8") as f: json.dump(cache, f, indent = 2, sort_keys = True) tmp.replace(path) except OSError: @@ -620,7 +620,7 @@ def _fetch_swa_entry_from_hf(repo_id: str) -> Optional[object]: repo_type = "model", cache_dir = active_hf_hub_cache(), ) - with open(cfg_path) as f: + with open(cfg_path, encoding = "utf-8") as f: cfg = json.load(f) except Exception: return None @@ -3596,7 +3596,7 @@ class LlamaCppBackend: except Exception: pass try: - with open("/proc/meminfo") as f: + with open("/proc/meminfo", encoding = "utf-8") as f: for line in f: if line.startswith("MemAvailable:"): return int(line.split()[1]) // 1024 # kB -> MiB @@ -9477,7 +9477,7 @@ class LlamaCppBackend: return try: path.parent.mkdir(parents = True, exist_ok = True) - path.write_text(f"{pid}:{cls._pid_start_identity(pid)}") + path.write_text(f"{pid}:{cls._pid_start_identity(pid)}", encoding = "utf-8") except Exception as e: logger.debug(f"Could not write llama-server pidfile: {e}") @@ -9611,7 +9611,7 @@ class LlamaCppBackend: pid = -1 identity = "" try: - pid_str, _, identity = path.read_text().strip().partition(":") + pid_str, _, identity = path.read_text(encoding = "utf-8").strip().partition(":") pid = int(pid_str) except Exception: pid = -1 diff --git a/studio/backend/core/inference/sandbox_site/sitecustomize.py b/studio/backend/core/inference/sandbox_site/sitecustomize.py index 244fa95145..a909bfbb90 100644 --- a/studio/backend/core/inference/sandbox_site/sitecustomize.py +++ b/studio/backend/core/inference/sandbox_site/sitecustomize.py @@ -111,7 +111,7 @@ def _load_sidecar(cwd): """Return the persisted ``source -> healed target`` map, or {} on any error (missing/corrupt/foreign sidecar degrades to in-process-only behaviour).""" try: - with open(_sidecar_path(cwd)) as fh: + with open(_sidecar_path(cwd), encoding = "utf-8") as fh: data = json.load(fh) except Exception: # noqa: BLE001 - a bad sidecar must never break user code return {} @@ -131,7 +131,7 @@ def _record_sidecar(cwd, source, target): return data[source] = target tmp = _sidecar_path(cwd) + ".tmp" - with open(tmp, "w") as fh: + with open(tmp, "w", encoding = "utf-8") as fh: json.dump(data, fh) os.replace(tmp, _sidecar_path(cwd)) except Exception: # noqa: BLE001 - persistence is best effort only diff --git a/studio/backend/core/inference/worker.py b/studio/backend/core/inference/worker.py index 367de196f7..254eda40a3 100644 --- a/studio/backend/core/inference/worker.py +++ b/studio/backend/core/inference/worker.py @@ -151,7 +151,7 @@ def _resolve_lora_4bit(mc, load_in_4bit: bool) -> bool: import json try: - with open(adapter_cfg_path) as f: + with open(adapter_cfg_path, encoding = "utf-8") as f: adapter_cfg = json.load(f) training_method = adapter_cfg.get("unsloth_training_method") if training_method == "lora" and load_in_4bit: @@ -961,7 +961,10 @@ def run_inference_process( if _local_adapter_cfg.is_file(): try: _lora_base = ( - _json.loads(_local_adapter_cfg.read_text()).get("base_model_name_or_path") or None + _json.loads(_local_adapter_cfg.read_text(encoding = "utf-8")).get( + "base_model_name_or_path" + ) + or None ) except Exception: _lora_base = None diff --git a/studio/backend/core/rag/embeddings.py b/studio/backend/core/rag/embeddings.py index 3354585d2a..c86c0d3c51 100644 --- a/studio/backend/core/rag/embeddings.py +++ b/studio/backend/core/rag/embeddings.py @@ -100,7 +100,7 @@ def _st_module_subdirs(name: str, token: str | None) -> tuple[str, ...]: path = Path(normalize_path(name)).expanduser() / "modules.json" if not path.is_file(): return () - data = json.loads(path.read_text()) + data = json.loads(path.read_text(encoding = "utf-8")) else: from huggingface_hub import hf_hub_download from huggingface_hub.utils import EntryNotFoundError @@ -115,7 +115,7 @@ def _st_module_subdirs(name: str, token: str | None) -> tuple[str, ...]: ) except EntryNotFoundError: return () - data = json.loads(open(local).read()) + data = json.loads(open(local, encoding = "utf-8").read()) subdirs = [] for module in data or (): sub = str((module or {}).get("path", "")).strip().strip("/") diff --git a/studio/backend/hub/services/models/ollama.py b/studio/backend/hub/services/models/ollama.py index 2ccdbb44f1..190aef0c71 100644 --- a/studio/backend/hub/services/models/ollama.py +++ b/studio/backend/hub/services/models/ollama.py @@ -215,7 +215,7 @@ def _ollama_model_info_from_manifest( return None try: - manifest = json.loads(tag_file.read_text()) + manifest = json.loads(tag_file.read_text(encoding = "utf-8")) except (json.JSONDecodeError, OSError) as e: logger.debug("Skipping unreadable/invalid Ollama manifest %s: %s", tag_file, e) return None @@ -228,7 +228,7 @@ def _ollama_model_info_from_manifest( config_blob = _ollama_blob_path(blobs_dir, config_digest) if config_blob is not None and _safe_is_file(config_blob): try: - cfg = json.loads(config_blob.read_text()) + cfg = json.loads(config_blob.read_text(encoding = "utf-8")) model_type = cfg.get("model_type", "") file_type = cfg.get("file_type", "") except (json.JSONDecodeError, OSError) as e: diff --git a/studio/backend/hub/utils/download_registry.py b/studio/backend/hub/utils/download_registry.py index 9e2b7d1a6d..243caab8f7 100644 --- a/studio/backend/hub/utils/download_registry.py +++ b/studio/backend/hub/utils/download_registry.py @@ -462,7 +462,7 @@ def _read_marker_value(marker: Path) -> Optional[str]: try: if not marker.exists(): return None - value = marker.read_text().strip() + value = marker.read_text(encoding = "utf-8").strip() except OSError: return None return value if value in VALID_TRANSPORTS else None @@ -473,7 +473,7 @@ def _write_marker_value(marker: Path, mode: str) -> None: # tmp + rename so a SIGKILL mid-write can't leave a half-written marker. # The tmp name is per-process so concurrent writers don't clobber tmps. tmp = marker.with_name(f"{marker.name}.tmp-{os.getpid()}") - tmp.write_text(mode) + tmp.write_text(mode, encoding = "utf-8") os.replace(tmp, marker) except OSError: # Best-effort: a missing marker next run purges the partial defensively, diff --git a/studio/backend/hub/utils/paths.py b/studio/backend/hub/utils/paths.py index 81621edcf9..7b9c46d32f 100644 --- a/studio/backend/hub/utils/paths.py +++ b/studio/backend/hub/utils/paths.py @@ -103,7 +103,7 @@ def _is_wsl() -> bool: if sys.platform == "win32": return False try: - return "microsoft" in Path("/proc/version").read_text().lower() + return "microsoft" in Path("/proc/version").read_text(encoding = "utf-8").lower() except Exception: return False @@ -124,7 +124,7 @@ def _wsl_automount_root() -> str: import configparser parser = configparser.ConfigParser(inline_comment_prefixes = ("#", ";")) - parser.read("/etc/wsl.conf") + parser.read("/etc/wsl.conf", encoding = "utf-8") root = parser.get("automount", "root", fallback = "").strip().strip("\"'") except Exception: return default diff --git a/studio/backend/main.py b/studio/backend/main.py index 4a8cab778d..bcf5c281df 100644 --- a/studio/backend/main.py +++ b/studio/backend/main.py @@ -254,7 +254,11 @@ def _read_studio_install_id() -> str: /api/health emits "" and the launcher accepts any healthy backend. Carries no install-path info (matters when Unsloth runs -H 0.0.0.0).""" try: - token = (_STUDIO_ROOT_RESOLVED / "share" / "studio_install_id").read_text().strip() + token = ( + (_STUDIO_ROOT_RESOLVED / "share" / "studio_install_id") + .read_text(encoding = "utf-8") + .strip() + ) except (OSError, ValueError): return "" return token if _STUDIO_INSTALL_ID_RE.fullmatch(token) else "" diff --git a/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/state_store.py b/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/state_store.py index 67107c285a..b4c226136b 100644 --- a/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/state_store.py +++ b/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/state_store.py @@ -20,7 +20,7 @@ class StateStore: self._data: Dict[str, Any] = {} if self.path.exists(): try: - with self.path.open() as f: + with self.path.open(encoding = "utf-8") as f: self._data = json.load(f) except Exception: self._data = {} @@ -51,7 +51,7 @@ class StateStore: def _flush(self) -> None: tmp = self.path.with_suffix(self.path.suffix + ".tmp") - with tmp.open("w") as f: + with tmp.open("w", encoding = "utf-8") as f: json.dump(self._data, f, indent = 2, default = str) os.replace(tmp, self.path) @@ -63,12 +63,14 @@ class JsonlWriter: self.path = Path(path) self.path.parent.mkdir(parents = True, exist_ok = True) self._lock = threading.Lock() - self._fh = self.path.open("a", buffering = 1) + self._fh = self.path.open("a", buffering = 1, encoding = "utf-8") self._count_seen_keys: set[str] = set() # Preload seen keys for dedup across resumes if self.path.exists() and self.path.stat().st_size > 0: try: - with self.path.open() as f: + # No guess is safe for a file an older build wrote in the + # operator's locale, so read past whatever will not decode. + with self.path.open(encoding = "utf-8", errors = "replace") as f: for line in f: try: obj = json.loads(line) diff --git a/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/impl.py b/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/impl.py index 7272e426ad..6016f5611f 100644 --- a/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/impl.py +++ b/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/impl.py @@ -27,7 +27,7 @@ class UnstructuredSeedReader(SeedReader[UnstructuredSeedSource]): orig_name = path_obj.name if meta_path.exists(): try: - meta = json_mod.loads(meta_path.read_text()) + meta = json_mod.loads(meta_path.read_text(encoding = "utf-8")) orig_name = meta.get("original_filename", path_obj.name) except (json_mod.JSONDecodeError, OSError): pass diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 7197483841..06911fd866 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -3778,7 +3778,7 @@ def _effective_load_in_4bit(config: ModelConfig, requested: bool) -> bool: if not adapter_cfg_path.exists(): return load_in_4bit try: - with open(adapter_cfg_path) as f: + with open(adapter_cfg_path, encoding = "utf-8") as f: adapter_cfg = json.load(f) if not isinstance(adapter_cfg, dict): # malformed -> keep requested return load_in_4bit diff --git a/studio/backend/routes/models.py b/studio/backend/routes/models.py index fd779590e6..dc850becf0 100644 --- a/studio/backend/routes/models.py +++ b/studio/backend/routes/models.py @@ -722,7 +722,7 @@ def _scan_ollama_dir(ollama_dir: Path, limit: Optional[int] = None) -> List[Loca stem_hash = hashlib.sha256(manifest_key.encode()).hexdigest()[:10] try: - manifest = json.loads(tag_file.read_text()) + manifest = json.loads(tag_file.read_text(encoding = "utf-8")) except (json.JSONDecodeError, OSError) as e: logger.debug( "Skipping unreadable/invalid Ollama manifest %s: %s", @@ -738,7 +738,7 @@ def _scan_ollama_dir(ollama_dir: Path, limit: Optional[int] = None) -> List[Loca config_blob = blobs_dir / config_digest.replace(":", "-") if config_blob.is_file(): try: - cfg = json.loads(config_blob.read_text()) + cfg = json.loads(config_blob.read_text(encoding = "utf-8")) model_type = cfg.get("model_type", "") file_type = cfg.get("file_type", "") except (json.JSONDecodeError, OSError) as e: @@ -1042,7 +1042,7 @@ def _dir_has_downloaded_model(directory: Path, max_entries: int = 4000) -> bool: if not m.is_file(): continue try: - manifest = json.loads(m.read_text()) + manifest = json.loads(m.read_text(encoding = "utf-8")) except (json.JSONDecodeError, OSError, ValueError): continue for layer in manifest.get("layers") or []: diff --git a/studio/backend/run.py b/studio/backend/run.py index 38636a6fba..2189388cf9 100644 --- a/studio/backend/run.py +++ b/studio/backend/run.py @@ -774,7 +774,7 @@ def _write_pid_file(): """Write the current process PID to the studio PID file.""" try: _PID_FILE.parent.mkdir(parents = True, exist_ok = True) - _PID_FILE.write_text(str(os.getpid())) + _PID_FILE.write_text(str(os.getpid()), encoding = "utf-8") except OSError: pass @@ -783,7 +783,7 @@ def _remove_pid_file(): """Remove the PID file if it belongs to this process.""" try: if _PID_FILE.is_file(): - stored = _PID_FILE.read_text().strip() + stored = _PID_FILE.read_text(encoding = "utf-8").strip() if stored == str(os.getpid()): _PID_FILE.unlink(missing_ok = True) except OSError: diff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.py index 5c9af51581..f3b968c8df 100644 --- a/studio/backend/utils/hardware/hardware.py +++ b/studio/backend/utils/hardware/hardware.py @@ -776,7 +776,7 @@ def _rocm_linux_sysfs_gpu_busy_pct() -> Optional[float]: files = glob.glob("/sys/class/drm/card*/device/gpu_busy_percent") if not files: return None - values = [int(open(f).read().strip()) for f in files] + values = [int(open(f, encoding = "utf-8").read().strip()) for f in files] return round(sum(values) / len(values), 1) except Exception: return None @@ -790,7 +790,7 @@ def _rocm_linux_sysfs_temp_c() -> Optional[float]: files = glob.glob("/sys/class/drm/card*/device/hwmon/hwmon*/temp1_input") if not files: return None - temps = [int(open(f).read().strip()) / 1000.0 for f in files] + temps = [int(open(f, encoding = "utf-8").read().strip()) / 1000.0 for f in files] return round(max(temps), 1) except Exception: return None @@ -807,7 +807,9 @@ def _rocm_linux_sysfs_power_w() -> Optional[float]: ): files = glob.glob(pattern) if files: - watts = sum(int(open(f).read().strip()) / 1_000_000.0 for f in files) + watts = sum( + int(open(f, encoding = "utf-8").read().strip()) / 1_000_000.0 for f in files + ) return round(watts, 1) return None except Exception: @@ -852,8 +854,8 @@ def _rocm_linux_sysfs_vram_gb() -> tuple[Optional[float], Optional[float]]: total_files = glob.glob("/sys/class/drm/card*/device/mem_info_vram_total") if not used_files or not total_files: return None, None - used_bytes = sum(int(open(f).read().strip()) for f in used_files) - total_bytes = sum(int(open(f).read().strip()) for f in total_files) + used_bytes = sum(int(open(f, encoding = "utf-8").read().strip()) for f in used_files) + total_bytes = sum(int(open(f, encoding = "utf-8").read().strip()) for f in total_files) if total_bytes == 0: return None, None return round(used_bytes / (1024**3), 2), round(total_bytes / (1024**3), 2) @@ -893,7 +895,7 @@ def _rocm_kfd_gpu_pci_ids() -> list[str]: continue props: dict[str, int] = {} try: - with open(os.path.join(node_dir, "properties")) as f: + with open(os.path.join(node_dir, "properties"), encoding = "utf-8") as f: for line in f: parts = line.split() if len(parts) == 2: @@ -979,9 +981,9 @@ def _rocm_linux_sysfs_vram_by_pci_gb() -> dict[str, tuple[float, float]]: if not bdf: continue try: - with open(os.path.join(dev_dir, "mem_info_vram_used")) as f: + with open(os.path.join(dev_dir, "mem_info_vram_used"), encoding = "utf-8") as f: used_bytes = int(f.read().strip()) - with open(os.path.join(dev_dir, "mem_info_vram_total")) as f: + with open(os.path.join(dev_dir, "mem_info_vram_total"), encoding = "utf-8") as f: total_bytes = int(f.read().strip()) except (OSError, ValueError): continue diff --git a/studio/backend/utils/models/checkpoints.py b/studio/backend/utils/models/checkpoints.py index f2125ad034..6950667bbd 100644 --- a/studio/backend/utils/models/checkpoints.py +++ b/studio/backend/utils/models/checkpoints.py @@ -129,7 +129,7 @@ def _read_checkpoint_loss(checkpoint_path: Path) -> Optional[float]: if not trainer_state.exists(): return None try: - with open(trainer_state) as f: + with open(trainer_state, encoding = "utf-8") as f: state = json.load(f) log_history = state.get("log_history", []) if log_history: @@ -174,18 +174,18 @@ def scan_checkpoints( metadata: dict = {} try: if adapter_config.exists(): - cfg = json.loads(adapter_config.read_text()) + cfg = json.loads(adapter_config.read_text(encoding = "utf-8")) metadata["base_model"] = cfg.get("base_model_name_or_path") metadata["peft_type"] = cfg.get("peft_type") metadata["lora_rank"] = cfg.get("r") elif config_file.exists(): - cfg = json.loads(config_file.read_text()) + cfg = json.loads(config_file.read_text(encoding = "utf-8")) metadata["base_model"] = cfg.get("_name_or_path") # Detect BNB quantization from config.json if config_file.exists(): if "cfg" not in dir(): - cfg = json.loads(config_file.read_text()) + cfg = json.loads(config_file.read_text(encoding = "utf-8")) quant_cfg = cfg.get("quantization_config") if ( isinstance(quant_cfg, dict) diff --git a/studio/backend/utils/models/model_config.py b/studio/backend/utils/models/model_config.py index 4897f05ce4..893b842e11 100644 --- a/studio/backend/utils/models/model_config.py +++ b/studio/backend/utils/models/model_config.py @@ -631,7 +631,7 @@ def _raw_config_has_vision_config( cache_dir = active_hf_hub_cache(), ) ) - config = json.loads(config_path.read_text()) + config = json.loads(config_path.read_text(encoding = "utf-8")) architectures = config.get("architectures") or [] model_type = config.get("model_type") explicit_vision = ( @@ -1083,7 +1083,7 @@ def _detect_audio_from_tokenizer( ]: tok_file = snapshot / tok_path if tok_file.exists(): - tok_config = json.loads(tok_file.read_text()) + tok_config = json.loads(tok_file.read_text(encoding = "utf-8")) read_any = True result = _check_token_patterns(tok_config) if result: @@ -2283,7 +2283,7 @@ def scan_exported_models( export_meta = run_dir / "export_metadata.json" try: if export_meta.exists(): - meta = json.loads(export_meta.read_text()) + meta = json.loads(export_meta.read_text(encoding = "utf-8")) base_model = meta.get("base_model") except Exception: pass @@ -2312,7 +2312,7 @@ def scan_exported_models( if adapter_config.exists(): export_type = "lora" try: - cfg = json.loads(adapter_config.read_text()) + cfg = json.loads(adapter_config.read_text(encoding = "utf-8")) base_model = cfg.get("base_model_name_or_path") except Exception: pass @@ -2321,7 +2321,7 @@ def scan_exported_models( export_meta = checkpoint_dir / "export_metadata.json" try: if export_meta.exists(): - meta = json.loads(export_meta.read_text()) + meta = json.loads(export_meta.read_text(encoding = "utf-8")) base_model = meta.get("base_model") except Exception: pass @@ -2334,7 +2334,7 @@ def scan_exported_models( export_meta = meta_dir / "export_metadata.json" try: if export_meta.exists(): - meta = json.loads(export_meta.read_text()) + meta = json.loads(export_meta.read_text(encoding = "utf-8")) base_model = meta.get("base_model") if base_model: break @@ -2354,7 +2354,7 @@ def scan_exported_models( outputs_adapter_cfg = resolve_output_dir(run_dir.name) / "adapter_config.json" try: if outputs_adapter_cfg.exists(): - cfg = json.loads(outputs_adapter_cfg.read_text()) + cfg = json.loads(outputs_adapter_cfg.read_text(encoding = "utf-8")) base_model = cfg.get("base_model_name_or_path") except Exception: pass @@ -2380,7 +2380,7 @@ def get_base_model_from_checkpoint(checkpoint_path: str) -> Optional[str]: adapter_config_path = checkpoint_path_obj / "adapter_config.json" if adapter_config_path.exists(): - with open(adapter_config_path, "r") as f: + with open(adapter_config_path, "r", encoding = "utf-8") as f: config = json.load(f) base_model = config.get("base_model_name_or_path") if base_model: @@ -2389,7 +2389,7 @@ def get_base_model_from_checkpoint(checkpoint_path: str) -> Optional[str]: config_path = checkpoint_path_obj / "config.json" if config_path.exists(): - with open(config_path, "r") as f: + with open(config_path, "r", encoding = "utf-8") as f: config = json.load(f) for key in ("model_name", "_name_or_path"): base_model = config.get(key) @@ -2445,7 +2445,7 @@ def get_base_model_from_lora(lora_path: str) -> Optional[str]: # adapter_config.json first adapter_config_path = lora_path_obj / "adapter_config.json" if adapter_config_path.exists(): - with open(adapter_config_path, "r") as f: + with open(adapter_config_path, "r", encoding = "utf-8") as f: config = json.load(f) base_model = config.get("base_model_name_or_path") if base_model: @@ -2535,7 +2535,7 @@ def get_base_model_from_lora_identifier( last_exc = exc continue try: - with open(cfg_path, "r") as f: + with open(cfg_path, "r", encoding = "utf-8") as f: base_model = json.load(f).get("base_model_name_or_path") except Exception as exc: logger.warning("Could not parse adapter_config.json for '%s': %s", identifier, exc) @@ -2781,7 +2781,7 @@ class ModelConfig: meta_path = gguf_dir / "export_metadata.json" if meta_path.exists(): try: - meta = json.loads(meta_path.read_text()) + meta = json.loads(meta_path.read_text(encoding = "utf-8")) base = meta.get("base_model") if base and is_vision_model(base, hf_token = hf_token): base_is_vision = True @@ -2912,7 +2912,7 @@ class ModelConfig: token = hf_token, cache_dir = active_hf_hub_cache(), ) - with open(config_path, "r") as f: + with open(config_path, "r", encoding = "utf-8") as f: adapter_config = json.load(f) base_model = adapter_config.get("base_model_name_or_path") if base_model: diff --git a/studio/backend/utils/paths/path_utils.py b/studio/backend/utils/paths/path_utils.py index 65541661f1..55fafeeb0e 100644 --- a/studio/backend/utils/paths/path_utils.py +++ b/studio/backend/utils/paths/path_utils.py @@ -34,7 +34,7 @@ def _is_wsl() -> bool: if sys.platform == "win32": return False try: - with open("/proc/version", "r") as f: + with open("/proc/version", "r", encoding = "utf-8") as f: return "microsoft" in f.read().lower() except Exception: return False diff --git a/studio/backend/utils/paths/storage_roots.py b/studio/backend/utils/paths/storage_roots.py index cea3cc61e3..ab888ec49e 100644 --- a/studio/backend/utils/paths/storage_roots.py +++ b/studio/backend/utils/paths/storage_roots.py @@ -212,7 +212,7 @@ def lmstudio_model_dirs() -> list[Path]: settings_path = Path.home() / ".lmstudio" / "settings.json" if settings_path.is_file(): try: - with open(settings_path) as f: + with open(settings_path, encoding = "utf-8") as f: settings = json.load(f) downloads = settings.get("downloadsFolder", "") if downloads: diff --git a/studio/backend/utils/security/consent.py b/studio/backend/utils/security/consent.py index fad52f21bb..6fee259139 100644 --- a/studio/backend/utils/security/consent.py +++ b/studio/backend/utils/security/consent.py @@ -142,7 +142,7 @@ def _load_remote_code_configs(model_name: str, hf_token: Optional[str] = None) - for name in _REMOTE_CODE_CONFIG_FILES: p = root / name if p.is_file(): - configs.append(json.loads(p.read_text())) + configs.append(json.loads(p.read_text(encoding = "utf-8"))) return configs from huggingface_hub import hf_hub_download @@ -164,7 +164,7 @@ def _load_remote_code_configs(model_name: str, hf_token: Optional[str] = None) - # Transient/auth failure is not "absent" -> fail closed to "unknown" so # the caller scans (a tokenizer/processor-only auto_map must not slip by). return None - configs.append(json.loads(Path(p).read_text())) + configs.append(json.loads(Path(p).read_text(encoding = "utf-8"))) # Every config was read or a genuine 404 -> an empty list is a definitive # "no auto_map", not "unknown". return configs diff --git a/studio/backend/utils/security/file_security.py b/studio/backend/utils/security/file_security.py index 3e12c15096..7724406e8d 100644 --- a/studio/backend/utils/security/file_security.py +++ b/studio/backend/utils/security/file_security.py @@ -199,7 +199,9 @@ def _indexed_shard_paths( inconclusive = True # transient: an index that might exist could not be read continue try: - weight_map = (json.loads(open(index_path).read()) or {}).get("weight_map") or {} + weight_map = (json.loads(open(index_path, encoding = "utf-8").read()) or {}).get( + "weight_map" + ) or {} for shard in weight_map.values(): shard_norm = _normalize_repo_path(str(shard)) # weight_map paths are relative to the index file's directory. @@ -326,7 +328,7 @@ def _st_load_roots(snapshot: Path) -> list: roots = [snapshot] try: import json - modules = json.loads((snapshot / "modules.json").read_text()) + modules = json.loads((snapshot / "modules.json").read_text(encoding = "utf-8")) except (OSError, ValueError): return roots # no / invalid modules.json -> snapshot root is the only load root for module in modules or (): diff --git a/studio/backend/utils/security/remote_code_approvals.py b/studio/backend/utils/security/remote_code_approvals.py index ee38ddec6f..f1baac6924 100644 --- a/studio/backend/utils/security/remote_code_approvals.py +++ b/studio/backend/utils/security/remote_code_approvals.py @@ -69,7 +69,7 @@ def approval_target_key(targets) -> str: def _load() -> dict: """Parsed store, or an empty skeleton on any error (fail-safe = re-prompt).""" try: - with open(_store_path()) as f: + with open(_store_path(), encoding = "utf-8") as f: data = json.load(f) # Validate the shape, not just the version: a hand-edited ``subjects`` that is not a # dict (e.g. ``[]``) would otherwise crash lookup/record instead of failing safe. @@ -92,7 +92,7 @@ def _save(data: dict) -> None: storage_roots.ensure_dir(path.parent) tmp = path.parent / f".{path.name}.tmp-{os.getpid()}" try: - with open(tmp, "w") as f: + with open(tmp, "w", encoding = "utf-8") as f: json.dump(data, f, indent = 2) try: os.chmod(tmp, 0o600) diff --git a/studio/backend/utils/security/remote_code_scan.py b/studio/backend/utils/security/remote_code_scan.py index 583dac94b6..d4d8003252 100644 --- a/studio/backend/utils/security/remote_code_scan.py +++ b/studio/backend/utils/security/remote_code_scan.py @@ -21,6 +21,8 @@ canonical scanner loads in-repo so the fallback never silently takes over. from __future__ import annotations import hashlib +import io +import tokenize import importlib.util import pathlib import re @@ -392,6 +394,18 @@ def scan_remote_code_files(files: dict[str, str]) -> ScanResult: return result +def _read_python_source(path) -> str: + """Decode a .py the way Python will execute it: a PEP 263 cookie + (`# coding: cp1252`) wins, so forcing utf-8 would scan something other than + what runs.""" + data = path.read_bytes() + try: + encoding = tokenize.detect_encoding(io.BytesIO(data).readline)[0] + except (SyntaxError, ValueError): + encoding = "utf-8" + return data.decode(encoding, errors = "replace") + + def remote_code_fingerprint(files: dict[str, str]) -> str: """Stable sha256 over the (sorted) file contents, for pinning consent.""" h = hashlib.sha256() @@ -430,7 +444,7 @@ def repo_remote_code_files(model_name: str, hf_token: Optional[str] = None) -> d # for an RCE gate (HIGH stays approvable; only CRITICAL hard-blocks). for p in root.rglob("*.py"): if p.is_file(): - files[str(p.relative_to(root))] = p.read_text(errors = "replace") + files[str(p.relative_to(root))] = _read_python_source(p) # A local config can still point auto_map at an EXTERNAL Hub repo # (owner/name--module.Class) that executes on load, so fetch it. Every config # that can declare auto_map is checked, so a custom processor's external code @@ -440,7 +454,7 @@ def repo_remote_code_files(model_name: str, hf_token: Optional[str] = None) -> d p = root / name if p.is_file(): try: - ext_refs |= _auto_map_refs(json.loads(p.read_text())) + ext_refs |= _auto_map_refs(json.loads(p.read_text(encoding = "utf-8"))) except Exception: pass if not _add_external_refs(files, ext_refs, hf_token, model_name): @@ -469,7 +483,7 @@ def repo_remote_code_files(model_name: str, hf_token: Optional[str] = None) -> d f"{model_name}: config {cfg_name} could not be fetched ({exc})" ) from exc try: - refs |= _auto_map_refs(json.loads(Path(cfg_path).read_text())) + refs |= _auto_map_refs(json.loads(Path(cfg_path).read_text(encoding = "utf-8"))) except Exception: pass own_refs = {fn for repo, fn in refs if repo is None} @@ -519,7 +533,7 @@ def repo_remote_code_files(model_name: str, hf_token: Optional[str] = None) -> d raise RemoteCodeUnscannable( f"{model_name}: present file {fn} could not be fetched ({exc})" ) from exc - files[fn] = Path(fp).read_text(errors = "replace") + files[fn] = _read_python_source(Path(fp)) # Code referenced from another repo executes too: scan it or fail closed. if not _add_external_refs(files, refs, hf_token, model_name): raise RemoteCodeUnscannable(f"{model_name}: external auto_map code unreachable") @@ -602,7 +616,7 @@ def external_auto_map_repos(model_name: str, hf_token: Optional[str] = None) -> if not p.is_file(): continue try: - refs = _auto_map_refs(json.loads(p.read_text())) + refs = _auto_map_refs(json.loads(p.read_text(encoding = "utf-8"))) except Exception: continue repos.update(repo for repo, _fn in refs if repo) @@ -624,7 +638,7 @@ def external_auto_map_repos(model_name: str, hf_token: Optional[str] = None) -> except Exception: continue try: - refs = _auto_map_refs(json.loads(Path(cfg_path).read_text())) + refs = _auto_map_refs(json.loads(Path(cfg_path).read_text(encoding = "utf-8"))) except Exception: continue repos.update(repo for repo, _fn in refs if repo) @@ -701,5 +715,5 @@ def _add_external_refs(files: dict, refs, hf_token, model_name: str) -> bool: exc, ) return False - files[f"{repo}--{fn}"] = Path(fp).read_text(errors = "replace") + files[f"{repo}--{fn}"] = _read_python_source(Path(fp)) return True diff --git a/studio/backend/utils/transformers_version.py b/studio/backend/utils/transformers_version.py index 475a096248..b0a2da0e66 100644 --- a/studio/backend/utils/transformers_version.py +++ b/studio/backend/utils/transformers_version.py @@ -420,7 +420,7 @@ def _resolve_base_model(model_name: str) -> str: adapter_cfg_path = local_path / "adapter_config.json" if _safe_is_file(adapter_cfg_path): try: - with open(adapter_cfg_path) as f: + with open(adapter_cfg_path, encoding = "utf-8") as f: cfg = json.load(f) base = cfg.get("base_model_name_or_path") if base: @@ -437,7 +437,7 @@ def _resolve_base_model(model_name: str) -> str: config_json_path = local_path / "config.json" if _safe_is_file(config_json_path): try: - with open(config_json_path) as f: + with open(config_json_path, encoding = "utf-8") as f: cfg = json.load(f) # Unsloth writes model_name, HF writes _name_or_path; skip a self-reference. for _key in ("model_name", "_name_or_path"): @@ -534,14 +534,19 @@ def _adapter_base_from_hf_cache(model_name: str) -> str | None: try: if ref_main.is_file(): candidates.append( - repo_dir / "snapshots" / ref_main.read_text().strip() / "adapter_config.json" + repo_dir + / "snapshots" + / ref_main.read_text(encoding = "utf-8").strip() + / "adapter_config.json" ) candidates += sorted( repo_dir.glob("snapshots/*/adapter_config.json"), key = _mtime, reverse = True ) for cfg_path in candidates: if cfg_path.is_file(): - base = json.loads(cfg_path.read_text()).get("base_model_name_or_path") + base = json.loads(cfg_path.read_text(encoding = "utf-8")).get( + "base_model_name_or_path" + ) return base or None except Exception as exc: logger.debug("HF cache adapter_config.json lookup failed for '%s': %s", model_name, exc) @@ -611,7 +616,7 @@ def _check_tokenizer_config_needs_v5(model_name: str, hf_token: str | None = Non local_tc = local_path / "tokenizer_config.json" if _safe_is_file(local_tc): try: - with open(local_tc) as f: + with open(local_tc, encoding = "utf-8") as f: data = json.load(f) tokenizer_class = data.get("tokenizer_class", "") result = tokenizer_class in _TRANSFORMERS_5_TOKENIZER_CLASSES @@ -688,7 +693,12 @@ def _config_json_from_hf_cache(model_name: str) -> dict | None: ref_main = repo_dir / "refs" / "main" try: if ref_main.is_file(): - candidates.append(repo_dir / "snapshots" / ref_main.read_text().strip() / "config.json") + candidates.append( + repo_dir + / "snapshots" + / ref_main.read_text(encoding = "utf-8").strip() + / "config.json" + ) # No refs/main (e.g. commit-pinned downloads): newest snapshot by mtime, not a stale # lexicographically-first SHA, matching what the Hub cache would actually load. candidates += sorted( @@ -696,7 +706,7 @@ def _config_json_from_hf_cache(model_name: str) -> dict | None: ) for cfg_path in candidates: if cfg_path.is_file(): - with open(cfg_path) as f: + with open(cfg_path, encoding = "utf-8") as f: return json.load(f) except Exception as exc: logger.debug("HF cache config.json lookup failed for '%s': %s", model_name, exc) @@ -721,7 +731,7 @@ def _load_config_json(model_name: str, hf_token: str | None = None) -> dict | No local_cfg = Path(model_name) / "config.json" if _safe_is_file(local_cfg): try: - with open(local_cfg) as f: + with open(local_cfg, encoding = "utf-8") as f: cfg = json.load(f) _config_json_cache[cache_key] = cfg return cfg @@ -1755,7 +1765,7 @@ def _venv_dir_is_valid(venv_dir: str, packages: tuple[str, ...]) -> bool: metadata = di / "METADATA" if not metadata.is_file(): continue - for line in metadata.read_text(errors = "replace").splitlines(): + for line in metadata.read_text(errors = "replace", encoding = "utf-8").splitlines(): if line.startswith("Version:"): installed_ver = line.split(":", 1)[1].strip() if installed_ver != pkg_version: @@ -2391,7 +2401,10 @@ def _llmcompressor_shadow_is_valid() -> bool: """True if the shadow dir exists with a marker matching the current pin fingerprint.""" marker = Path(_VENV_LLMCOMPRESSOR_DIR) / _LLMC_SHADOW_MARKER try: - return marker.is_file() and marker.read_text().strip() == _LLMC_SHADOW_FINGERPRINT + return ( + marker.is_file() + and marker.read_text(encoding = "utf-8").strip() == _LLMC_SHADOW_FINGERPRINT + ) except Exception: return False @@ -2460,7 +2473,7 @@ def _ensure_venv_llmcompressor_exists() -> bool: if result.returncode == 0: try: (Path(_VENV_LLMCOMPRESSOR_DIR) / _LLMC_SHADOW_MARKER).write_text( - _LLMC_SHADOW_FINGERPRINT + _LLMC_SHADOW_FINGERPRINT, encoding = "utf-8" ) except Exception: pass diff --git a/studio/backend/utils/utils.py b/studio/backend/utils/utils.py index ce3d6704b7..2d9306f6ed 100644 --- a/studio/backend/utils/utils.py +++ b/studio/backend/utils/utils.py @@ -108,7 +108,7 @@ def hf_cache_snapshot_dir(model_name: str) -> Optional[Path]: ref = repo_dir / "refs" / "main" if not ref.is_file(): continue - commit = ref.read_text().strip() + commit = ref.read_text(encoding = "utf-8").strip() if not commit: continue snapshot = repo_dir / "snapshots" / commit diff --git a/studio/install_llama_prebuilt.py b/studio/install_llama_prebuilt.py index 676933b67c..47ad4bfc66 100644 --- a/studio/install_llama_prebuilt.py +++ b/studio/install_llama_prebuilt.py @@ -2504,7 +2504,7 @@ def detect_host() -> HostInfo: if is_linux: for _vendor_file in glob.glob("/sys/class/drm/card*/device/vendor"): try: - with open(_vendor_file) as _vf: + with open(_vendor_file, encoding = "utf-8") as _vf: if _vf.read().strip().lower() == "0x8086": has_intel_gpu = True break @@ -3050,7 +3050,7 @@ def _detect_host_rocm_version() -> tuple[int, int] | None: os.path.join(rocm_root, "lib", "rocm_version"), ): try: - with open(path) as fh: + with open(path, encoding = "utf-8") as fh: parts = fh.read().strip().split("-")[0].split(".") # Explicit length guard avoids relying on the broad except # below to swallow IndexError when the version file contains @@ -5526,7 +5526,9 @@ def write_prebuilt_metadata( "prebuilt_fallback_used": prebuilt_fallback_used, "installed_at_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), } - (install_dir / "UNSLOTH_PREBUILT_INFO.json").write_text(json.dumps(metadata, indent = 2) + "\n") + (install_dir / "UNSLOTH_PREBUILT_INFO.json").write_text( + json.dumps(metadata, indent = 2) + "\n", encoding = "utf-8" + ) def sync_marker_force_cpu(install_dir: Path, persist_force_cpu: bool) -> None: @@ -5537,13 +5539,13 @@ def sync_marker_force_cpu(install_dir: Path, persist_force_cpu: bool) -> None: GPU/Vulkan bundle that revives the crash (#7213).""" marker_path = install_dir / "UNSLOTH_PREBUILT_INFO.json" try: - marker = json.loads(marker_path.read_text()) + marker = json.loads(marker_path.read_text(encoding = "utf-8")) except (OSError, ValueError): return if not isinstance(marker, dict) or bool(marker.get("force_cpu")) == persist_force_cpu: return marker["force_cpu"] = persist_force_cpu - marker_path.write_text(json.dumps(marker, indent = 2) + "\n") + marker_path.write_text(json.dumps(marker, indent = 2) + "\n", encoding = "utf-8") log(f"existing install reused; recorded force_cpu={persist_force_cpu} from this run") diff --git a/studio/install_node_prebuilt.py b/studio/install_node_prebuilt.py index fb40634e95..948743c63c 100644 --- a/studio/install_node_prebuilt.py +++ b/studio/install_node_prebuilt.py @@ -535,7 +535,7 @@ def install_lock(lock_path: Path) -> Iterator[None]: break except FileExistsError: try: - raw = lock_path.read_text().strip() + raw = lock_path.read_text(encoding = "utf-8").strip() except FileNotFoundError: continue stale = False @@ -660,7 +660,7 @@ def write_metadata(install_dir: Path, *, version: str, asset: str, sha256: str) "asset": asset, "sha256": sha256, } - metadata_path(install_dir).write_text(json.dumps(payload, indent = 2) + "\n") + metadata_path(install_dir).write_text(json.dumps(payload, indent = 2) + "\n", encoding = "utf-8") def load_metadata(install_dir: Path) -> dict | None: @@ -668,7 +668,7 @@ def load_metadata(install_dir: Path) -> dict | None: if not path.exists(): return None try: - data = json.loads(path.read_text()) + data = json.loads(path.read_text(encoding = "utf-8")) except (json.JSONDecodeError, OSError): return None return data if isinstance(data, dict) else None diff --git a/studio/install_python_stack.py b/studio/install_python_stack.py index 8c33cb6ce9..5e9adb79d1 100644 --- a/studio/install_python_stack.py +++ b/studio/install_python_stack.py @@ -503,7 +503,7 @@ def _detect_rocm_version() -> tuple[int, int] | None: os.path.join(rocm_root, "lib", "rocm_version"), ): try: - with open(path) as fh: + with open(path, encoding = "utf-8") as fh: parts = fh.read().strip().split("-")[0].split(".") # Explicit length guard: don't rely on the broad except below to # swallow IndexError on a single-component version (e.g. "6\n"). @@ -852,9 +852,9 @@ def _linux_amd_display_device_present() -> bool: try: for dev in Path("/sys/bus/pci/devices").iterdir(): try: - if (dev / "vendor").read_text().strip() != "0x1002": + if (dev / "vendor").read_text(encoding = "utf-8").strip() != "0x1002": continue - if (dev / "class").read_text().strip().startswith("0x03"): + if (dev / "class").read_text(encoding = "utf-8").strip().startswith("0x03"): return True except OSError: continue @@ -1067,7 +1067,7 @@ def _has_rocm_gpu() -> bool: for entry in os.listdir(kfd_nodes): gpu_id_path = os.path.join(kfd_nodes, entry, "gpu_id") try: - with open(gpu_id_path) as fh: + with open(gpu_id_path, encoding = "utf-8") as fh: gpu_id = fh.read().strip() except OSError: continue @@ -1079,7 +1079,7 @@ def _has_rocm_gpu() -> bool: # false positive (e.g. NVIDIA open-driver KFD nodes lacking it). props_path = os.path.join(kfd_nodes, entry, "properties") try: - with open(props_path) as fh: + with open(props_path, encoding = "utf-8") as fh: props = fh.read() except OSError: continue # can't confirm vendor -- skip diff --git a/studio/prebuilt_core.py b/studio/prebuilt_core.py index 5dc85af1c2..1f711d14f0 100644 --- a/studio/prebuilt_core.py +++ b/studio/prebuilt_core.py @@ -1078,7 +1078,7 @@ def install_lock(lock_path: Path) -> Iterator[None]: except FileExistsError: stale = False try: - raw = lock_path.read_text().strip() + raw = lock_path.read_text(encoding = "utf-8").strip() except FileNotFoundError: # Lock vanished between our open and read -- retry continue diff --git a/tests/test_runtime_text_encoding.py b/tests/test_runtime_text_encoding.py new file mode 100644 index 0000000000..42cc4ddcc4 --- /dev/null +++ b/tests/test_runtime_text_encoding.py @@ -0,0 +1,407 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Guard: shipping code must name an encoding on every text read and write. + +`Path.read_text()`, `Path.write_text()`, `Path.open()` and builtin `open()` fall back +to `locale.getencoding()`: UTF-8 on the Linux and macOS runners, cp1252 on a stock +Windows install. Every file this repo reads at runtime is UTF-8 (HF `config.json` / +`tokenizer_config.json` / `adapter_config.json`, Ollama manifests, GGUF export +metadata), so on Windows those reads crash or, worse, succeed with mojibake: a +DeepSeek or Qwen tokenizer_config.json carries U+FF5C and U+2581 in its chat +template, and at utils/models/model_config.py that read sits inside a broad +`except Exception: logger.debug(...)`, so the token-pattern check silently +returned the wrong answer. + +Unlike the import-time rule in test_source_read_encoding.py this is scope agnostic: +runtime reads live inside functions, and shipping code has no legitimate reason to +let the operator's locale decide. No reachability analysis to get wrong, so no +allowlist and no false positives. + +Binary handles are skipped (no encoding to name, and passing one is a ValueError), +and a non-constant mode counts as unknown rather than text: demanding `encoding =` +on a call that may resolve to "rb" would leave no compliant way to write it. + +Known limitation, deliberately not closed: `configparser.ConfigParser.read()` also +defaults to the locale encoding, but cannot be matched by name without resolving the +receiver, since `f.read(n)`, `resp.read(limit)` and `handle.read(chunk)` are spelled +identically. Flagging it would be a false positive with no compliant fix, the exact +failure mode this guard avoids. The one live `ConfigParser.read` (/etc/wsl.conf, +hub/utils/paths.py) is pinned by hand; a future one has to be caught in review. +""" + +# `str | None` below is evaluated at import on Python 3.9 (requires-python >= 3.9). +from __future__ import annotations + +import ast +import subprocess +from pathlib import Path + + +REPO = Path(__file__).resolve().parent.parent +# Everything that ships. `studio/` covers the installers too: install_python_stack.py +# reads /sys/class/kfd, the same detection path as utils/hardware/hardware.py. Test +# trees fall under the narrower import-time rule in test_source_read_encoding.py. +ROOTS = (REPO / "unsloth", REPO / "studio", REPO / "unsloth_cli") +# The frontend tree is TypeScript; node_modules is vendored third-party code. +SKIP_DIRS = {"build", "dist", "frontend", "node_modules", "src-tauri", ".venv", "site-packages"} +GUARDED_METHODS = {"read_text", "write_text"} +# Path classes, so an unbound `Path.open(p)` shifts every argument one right. +PATH_CLASSES = {"Path", "PosixPath", "PurePath", "WindowsPath"} +# Values that re-select the platform default when passed as the encoding. +PLATFORM_DEFAULT_ENCODINGS = (None, "locale") +# Calls that return the platform default, so naming one pins nothing. +PLATFORM_DEFAULT_CALLS = {"getdefaultencoding", "getencoding", "getpreferredencoding"} +# Modules whose `open` IS the builtin: same signature, same platform default. +BUILTIN_OPEN_MODULES = {"builtins", "io"} +# Take an encoding in "t" mode but default to "rb". Value is its positional slot. +COMPRESSED_OPENERS = {"bz2": 3, "gzip": 3, "lzma": None} +# Distinct from None so that "no mode argument at all" still means text. +UNKNOWN_MODE = object() + + +def _mode(call: ast.Call, positional_index: int): + """The call's mode, or UNKNOWN_MODE when it is not a literal.""" + # A splat hides the mode, so it is unknown rather than absent: falling through to + # "r" would flag a call that may resolve to binary, with no compliant way to fix it. + if any(isinstance(a, ast.Starred) for a in call.args): + return UNKNOWN_MODE + if any(kw.arg is None for kw in call.keywords): + return UNKNOWN_MODE + if len(call.args) > positional_index: + node = call.args[positional_index] + return node.value if isinstance(node, ast.Constant) else UNKNOWN_MODE + for kw in call.keywords: + if kw.arg == "mode": + return kw.value.value if isinstance(kw.value, ast.Constant) else UNKNOWN_MODE + return "r" + + +def _names_encoding(call: ast.Call) -> bool: + """True only for an encoding that actually pins one. + + `encoding = None` and `encoding = "locale"` re-select the platform default, so the + keyword being present is not enough. A `**kwargs` splat may carry an encoding we + cannot see, so it counts as named rather than as an unsatisfiable demand. + """ + for kw in call.keywords: + if kw.arg is None: + return True + if kw.arg != "encoding": + continue + if isinstance(kw.value, ast.Constant) and kw.value.value in PLATFORM_DEFAULT_ENCODINGS: + return False + if isinstance(kw.value, ast.Call) and _callee_name(kw.value.func) in PLATFORM_DEFAULT_CALLS: + return False # locale.getencoding() is the default, spelled out + return True + return False + + +def _is_text(call: ast.Call, positional_index: int) -> bool: + mode = _mode(call, positional_index) + return mode is not UNKNOWN_MODE and "b" not in str(mode) + + +def _imports_at_each_call(tree: ast.Module) -> dict: + """The imports visible at every call, keyed by node id. + + A function's own imports stay in that function: hoisting them would let one local + `from PIL.Image import open` turn off the builtin check for the whole file. + """ + visible_at = {} + + def walk(node, visible): + if isinstance(node, ast.Call): + visible_at[id(node)] = visible + for child in ast.iter_child_nodes(node): + if isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)): + walk(child, {**visible, **_imported_names(child)}) + else: + walk(child, visible) + + walk(tree, _imported_names(tree)) + return visible_at + + +def _foreign_names(tree: ast.Module) -> set: + """Names bound to an object another library built. + + `z = zipfile.ZipFile(p)` then `z.open(name)` is a binary member stream taking no + encoding, so demanding one leaves no correct edit. + """ + modules = _imported_names(tree) + names = set() + for node in ast.walk(tree): + if not isinstance(node, ast.Assign) or not isinstance(node.value, ast.Call): + continue + if _foreign_receiver(node.value, modules): + names.update(t.id for t in node.targets if isinstance(t, ast.Name)) + return names + + +def _imported_names(tree) -> dict: + """Names this module's imports bind, mapped to where they came from. + + The name alone settles nothing: `import tarfile as tf` hides an opener that takes + no encoding, and `from PIL.Image import open` puts another behind the most familiar + name there is. Resolving the origin covers both, with no module list to maintain. + """ + bound = {} + stack = list(ast.iter_child_nodes(tree)) + while stack: + node = stack.pop() + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)): + continue # that function's business, not this scope's + if isinstance(node, ast.Import): + for a in node.names: + bound[(a.asname or a.name).split(".")[0]] = a.name + elif isinstance(node, ast.ImportFrom): + for a in node.names: + bound[a.asname or a.name] = f"{node.module}.{a.name}" if node.module else a.name + else: + stack.extend(ast.iter_child_nodes(node)) + return bound + + +def _callee_name(func): + """The bare name a callee ends in, whether or not it is qualified.""" + return func.id if isinstance(func, ast.Name) else getattr(func, "attr", None) + + +def _origin_root(name, modules) -> str: + """The top-level module a bound name came from, or the name itself.""" + return modules.get(name, name).split(".")[0] + + +def _compressed_key(name, modules): + """The COMPRESSED_OPENERS entry this receiver resolves to, if any.""" + for candidate in (name, _origin_root(name, modules)): + if candidate in COMPRESSED_OPENERS: + return candidate + return None + + +def _open_alias(name, modules): + """What a bare callable resolves to: "builtin", a COMPRESSED_OPENERS key, or None.""" + origin = modules.get(name) + if origin is None: + return "builtin" if name == "open" else None + parts = origin.split(".") + if parts[-1] != "open": + return None + if parts[0] in BUILTIN_OPEN_MODULES or origin == "open": + return "builtin" + return parts[0] if parts[0] in COMPRESSED_OPENERS else None + + +def _is_path_class(name, modules) -> bool: + """True for a pathlib class, including under an alias.""" + if name is None: + return False + return (modules.get(name) or name).split(".")[-1] in PATH_CLASSES + + +def _is_path_attr(node) -> bool: + """True for a qualified path class, as in `pathlib.Path`.""" + return isinstance(node, ast.Attribute) and node.attr in PATH_CLASSES + + +def _foreign_receiver(node, modules) -> bool: + """True when the thing before `.open` is an object another library built. + + `zipfile.ZipFile(p).open(name)` returns a binary member stream taking no encoding, + so it needs the same exemption as the bare `zipfile.open` spelling. + """ + if not isinstance(node, ast.Call): + return False + func = node.func + if isinstance(func, ast.Attribute) and isinstance(func.value, ast.Name): + root, name = func.value.id, func.attr + elif isinstance(func, ast.Name): + root = name = func.id + else: + return False + return root in modules and not _is_path_class(name, modules) + + +def _offender( + call: ast.Call, + modules = None, + foreign = (), +) -> str | None: + """The call's name if it does text I/O without pinning an encoding.""" + modules = {} if modules is None else modules + func = call.func + if isinstance(func, ast.Attribute): + receiver = func.value.id if isinstance(func.value, ast.Name) else None + # `Path.read_text(p)` is `p.read_text()` unbound: the instance takes slot 0, + # so every argument shifts one place right. + shift = 1 if _is_path_class(receiver, modules) or _is_path_attr(func.value) else 0 + if func.attr in GUARDED_METHODS: + if func.attr == "read_text" and not shift and call.args: + first = call.args[0] + # Bound read_text takes encoding first, so None or "locale" there is a + # platform-default read. Any other positional means the receiver is + # importlib.metadata's Distribution: a filename, and no encoding at all. + if isinstance(first, ast.Constant) and first.value in PLATFORM_DEFAULT_ENCODINGS: + return "read_text()" + return None + return None if _names_encoding(call) else f"{func.attr}()" + if func.attr == "open": + if receiver is not None and _origin_root(receiver, modules) in BUILTIN_OPEN_MODULES: + return ( + None if not _is_text(call, 1) or _names_encoding(call) else f"{receiver}.open()" + ) + compressed = _compressed_key(receiver, modules) if receiver else None + if compressed is not None: + # "rb" by default, so only an explicit text mode is in scope. + mode = _mode(call, 1) + if mode is UNKNOWN_MODE or "t" not in str(mode): + return None + return None if _names_encoding(call) else f"{compressed}.open()" + # Any other imported receiver is somebody else's opener: tarfile takes a + # compression mode, Image a binary file. Neither has an encoding to name. + if receiver is not None and receiver in modules and receiver not in PATH_CLASSES: + return None + if _foreign_receiver(func.value, modules) or receiver in foreign: + return None + if not _is_text(call, shift): + return None + return None if _names_encoding(call) else "Path.open()" + return None + if isinstance(func, ast.Name): + alias = _open_alias(func.id, modules) + if alias == "builtin": + if not _is_text(call, 1): + return None + return None if _names_encoding(call) else "open()" + if alias is not None: + mode = _mode(call, 1) + if mode is UNKNOWN_MODE or "t" not in str(mode): + return None + return None if _names_encoding(call) else f"{alias}.open()" + return None + + +def _is_test_path(path: Path) -> bool: + parts = path.relative_to(REPO).parts + if SKIP_DIRS.intersection(parts): + return True + if "tests" in parts or "test" in parts: + return True + return path.name.startswith("test_") or path.name.endswith("_test.py") + + +def _offenders_in(src: str, label: str = ""): + tree = ast.parse(src, filename = label) + visible_at = _imports_at_each_call(tree) + foreign = _foreign_names(tree) + found = [] + for node in ast.walk(tree): + if isinstance(node, ast.Call): + name = _offender(node, visible_at.get(id(node), {}), foreign) + if name is not None: + found.append((node.lineno, name)) + return found + + +def _tracked_sources(): + """Shipping *.py that git is actually tracking. + + A walk also picks up whatever is lying in the checkout (a built `build/lib` copy, + a nested worktree, a vendored dep). None of those are ours to police, and a stale + artifact would fail this for everybody who has one. + """ + listed = subprocess.run( + ["git", "-C", str(REPO), "ls-files", "-z", "--", "*.py"], + capture_output = True, + timeout = 60, + ) + if listed.returncode != 0: + return None # not a checkout, so fall back to walking + names = listed.stdout.decode("utf-8", errors = "replace").split("\0") + return [REPO / n for n in names if n] + + +def _walked_sources(): + return [p for root in ROOTS if root.is_dir() for p in sorted(root.rglob("*.py"))] + + +def test_shipping_code_names_an_encoding(): + offenders = [] + sources = _tracked_sources() + if sources is None: + sources = _walked_sources() + roots = {r.resolve() for r in ROOTS} + for path in sorted(sources): + if not roots.intersection(path.resolve().parents) or _is_test_path(path): + continue + try: + tree = ast.parse(path.read_text(encoding = "utf-8"), filename = str(path)) + except SyntaxError: + continue + rel = path.relative_to(REPO).as_posix() + visible_at = _imports_at_each_call(tree) + foreign = _foreign_names(tree) + for node in ast.walk(tree): + if isinstance(node, ast.Call): + name = _offender(node, visible_at.get(id(node), {}), foreign) + if name is not None: + offenders.append(f"{rel}:{node.lineno}: {name}") + assert offenders == [], ( + f"{len(offenders)} text read/write call sites in shipping code let the " + "operator's locale decide the encoding, so they crash or silently " + 'produce mojibake on Windows. Pass encoding = "utf-8": ' + repr(offenders) + ) + + +# The assertion above passes vacuously once the trees are clean, so it cannot tell a +# working detector from one that always returns None. These pin the detector itself. + + +def test_detects_the_plain_cases(): + assert _offenders_in("from pathlib import Path\np = Path('x')\ns = p.read_text()\n") + assert _offenders_in("p.write_text('hi')\n") + assert _offenders_in("f = open('x')\n") + assert _offenders_in("f = open('x', 'w')\n") + assert _offenders_in("f = p.open()\n") + # Inside a function body too: shipping reads are not import-time. + assert _offenders_in("def load(p):\n return p.read_text()\n") + + +def test_rejects_encoding_that_reselects_the_platform_default(): + assert _offenders_in("s = p.read_text(encoding = None)\n") + assert _offenders_in("s = p.read_text(encoding = 'locale')\n") + + +def test_accepts_a_pinned_encoding(): + assert not _offenders_in("s = p.read_text(encoding = 'utf-8')\n") + assert not _offenders_in("f = open('x', 'w', encoding = 'utf-8')\n") + assert not _offenders_in("f = p.open(encoding = 'utf-8')\n") + assert not _offenders_in("s = p.read_text(encoding = 'utf-8', errors = 'replace')\n") + + +def test_skips_binary_handles(): + # Binary has no encoding to name; passing one is a ValueError. + assert not _offenders_in("f = open('x', 'rb')\n") + assert not _offenders_in("f = open('x', mode = 'wb')\n") + assert not _offenders_in("f = p.open('rb')\n") + + +def test_skips_unknown_modes(): + # A call that may resolve to "rb" has no compliant way to name an encoding. + assert not _offenders_in("mode = 'rb' if binary else 'r'\nf = open(path, mode)\n") + assert not _offenders_in("f = open(path, mode = chosen)\n") + + +def test_skips_foreign_openers_and_readers(): + assert not _offenders_in("import fitz\nd = fitz.open(stream = b, filetype = 'pdf')\n") + assert not _offenders_in("import tarfile\nt = tarfile.open(p, 'r:gz')\n") + # importlib.metadata Distribution.read_text takes a positional filename. + assert not _offenders_in("s = dist.read_text('direct_url.json')\n") + + +def test_test_trees_are_out_of_scope(): + assert _is_test_path(REPO / "tests" / "test_x.py") + assert _is_test_path(REPO / "studio" / "backend" / "tests" / "helpers.py") + assert not _is_test_path(REPO / "studio" / "backend" / "routes" / "inference.py") diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 0b3a3698c2..dfb8082c46 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -2471,7 +2471,7 @@ def _get_statistics(statistics = None, force_download = True): for vendor_file in vendor_files: path = Path(vendor_file) if path.is_file(): - file_content = path.read_text().lower() + file_content = path.read_text(encoding = "utf-8").lower() if "amazon" in file_content: return "aws" elif "microsoft corporation" in file_content: diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index 13342157b0..5dcbb47ac3 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -1585,7 +1585,7 @@ class FastModel(FastBaseModel): if do_logging: redirector = contextlib.nullcontext() else: - redirector = contextlib.redirect_stdout(open(os.devnull, "w")) + redirector = contextlib.redirect_stdout(open(os.devnull, "w", encoding = "utf-8")) model_types = ["siglip"] + model_types # Set forced float32 env flag diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py index 7661b0d714..7fd8cd66b4 100644 --- a/unsloth/models/loader_utils.py +++ b/unsloth/models/loader_utils.py @@ -450,7 +450,7 @@ def _load_fp8_weight_map( index_path = None if index_path is not None: import json - with open(index_path, "r") as f: + with open(index_path, "r", encoding = "utf-8") as f: return json.load(f).get("weight_map", None) # Unsharded single file: map every tensor to it. diff --git a/unsloth/models/sentence_transformer.py b/unsloth/models/sentence_transformer.py index 006dbf6c3d..990521677d 100644 --- a/unsloth/models/sentence_transformer.py +++ b/unsloth/models/sentence_transformer.py @@ -2329,7 +2329,7 @@ def _patch_st_trainer_load_from_checkpoint(): if not os.path.isfile(modules_json): raise RuntimeError("Unsloth: PEFT checkpoint is missing modules.json.") try: - with open(modules_json, "r") as f: + with open(modules_json, "r", encoding = "utf-8") as f: module_configs = json.load(f) except Exception as e: raise RuntimeError("Unsloth: Cannot parse checkpoint modules.json.") from e diff --git a/unsloth_cli/_inference.py b/unsloth_cli/_inference.py index a2b0f9c04f..32b1694129 100644 --- a/unsloth_cli/_inference.py +++ b/unsloth_cli/_inference.py @@ -98,7 +98,7 @@ def _json_rank_count_from_env(name: str) -> Optional[int]: if value.lstrip().startswith(("[", "{")): data = json.loads(value) else: - with open(value, "r") as f: + with open(value, "r", encoding = "utf-8") as f: data = json.load(f) except (OSError, json.JSONDecodeError): return None @@ -158,7 +158,7 @@ def quiet_if_nonzero_mlx_rank(): sys.stderr.flush() saved_stdout_fd = os.dup(1) saved_stderr_fd = os.dup(2) - with open(os.devnull, "w") as devnull: + with open(os.devnull, "w", encoding = "utf-8") as devnull: try: os.dup2(devnull.fileno(), 1) os.dup2(devnull.fileno(), 2) diff --git a/unsloth_cli/commands/studio.py b/unsloth_cli/commands/studio.py index c41963aab9..323e713078 100644 --- a/unsloth_cli/commands/studio.py +++ b/unsloth_cli/commands/studio.py @@ -719,7 +719,7 @@ def _cli_update_password(conn: sqlite3.Connection, username: str, new_password: # credential after a later reset-password deletes auth.db. Mirrors # backend clear_bootstrap_password(). try: - stale_path.write_text("") + stale_path.write_text("", encoding = "utf-8") cleared = True except OSError: cleared = False @@ -2406,7 +2406,7 @@ def stop(): typer.echo("No running Unsloth server found (no PID file).") raise typer.Exit(0) - pid_text = _PID_FILE.read_text().strip() + pid_text = _PID_FILE.read_text(encoding = "utf-8").strip() if not pid_text.isdigit(): typer.echo(f"Invalid PID file contents: {pid_text}") _PID_FILE.unlink(missing_ok = True) @@ -2863,7 +2863,7 @@ def reset_password(): path.unlink(missing_ok = True) except OSError: try: - path.write_text("") + path.write_text("", encoding = "utf-8") except OSError as exc: typer.echo( f"Error: could not remove or clear {path.name} ({exc}); delete "