diff --git a/studio/backend/auth/storage.py b/studio/backend/auth/storage.py index 5f80ad89a3..35135b21eb 100644 --- a/studio/backend/auth/storage.py +++ b/studio/backend/auth/storage.py @@ -76,7 +76,13 @@ def _load_bootstrap_password() -> Optional[str]: global _bootstrap_password _bootstrap_password = None if _BOOTSTRAP_PW_PATH.is_file(): - bootstrap_password = _BOOTSTRAP_PW_PATH.read_text(encoding = "utf-8").strip() + # No caller handles a raise, so an unreadable file has to mean "no bootstrap + # password", not a dead backend. We write UTF-8, so bytes that will not + # decode are damage whose plaintext is worthless anyway. + try: + bootstrap_password = _BOOTSTRAP_PW_PATH.read_text(encoding = "utf-8").strip() + except (OSError, UnicodeDecodeError): + return _bootstrap_password if bootstrap_password: _bootstrap_password = bootstrap_password return _bootstrap_password diff --git a/studio/backend/cloudflare_tunnel.py b/studio/backend/cloudflare_tunnel.py index 78fce0c70a..f7967e2faa 100644 --- a/studio/backend/cloudflare_tunnel.py +++ b/studio/backend/cloudflare_tunnel.py @@ -310,6 +310,7 @@ class CloudflareTunnel: stderr = subprocess.STDOUT, stdin = subprocess.DEVNULL, text = True, + encoding = "utf-8", errors = "replace", bufsize = 1, **_windows_hidden_kwargs(), diff --git a/studio/backend/core/data_recipe/local_callable_validators.py b/studio/backend/core/data_recipe/local_callable_validators.py index ffc81669ae..143895d781 100644 --- a/studio/backend/core/data_recipe/local_callable_validators.py +++ b/studio/backend/core/data_recipe/local_callable_validators.py @@ -257,6 +257,8 @@ def _run_oxc_batch( cwd = str(_OXC_TOOL_DIR), input = json.dumps(payload), text = True, + encoding = "utf-8", + errors = "replace", capture_output = True, check = False, env = env, diff --git a/studio/backend/core/inference/inference.py b/studio/backend/core/inference/inference.py index 0af37e627f..e78bf1be8d 100644 --- a/studio/backend/core/inference/inference.py +++ b/studio/backend/core/inference/inference.py @@ -567,7 +567,7 @@ class InferenceBackend: _meta_path = Path(config.path) / "export_metadata.json" try: if _meta_path.exists(): - _meta = json.loads(_meta_path.read_text(encoding = "utf-8")) + _meta = json.loads(_meta_path.read_text(encoding = "utf-8-sig")) if _meta.get("base_model"): processor_source = _meta["base_model"] except Exception: diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 144aa1fd37..dcfbfb3338 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -85,6 +85,7 @@ from core.tool_healing import ( strip_outside_think, ) from utils.native_path_leases import child_env_without_native_path_secret +from utils.child_stdio import utf8_child_env from utils.hf_xet_fallback import hf_hub_download_with_xet_fallback from utils.subprocess_compat import ( windows_hidden_subprocess_kwargs as _windows_hidden_subprocess_kwargs, @@ -581,7 +582,7 @@ def _load_swa_cache() -> dict: if _SWA_CACHE is not None: return _SWA_CACHE try: - with open(_swa_cache_path(), encoding = "utf-8") as f: + with open(_swa_cache_path(), encoding = "utf-8-sig") as f: _SWA_CACHE = json.load(f) if not isinstance(_SWA_CACHE, dict): _SWA_CACHE = {} @@ -632,7 +633,7 @@ def _fetch_swa_entry_from_hf(repo_id: str) -> Optional[object]: repo_type = "model", cache_dir = active_hf_hub_cache(), ) - with open(cfg_path, encoding = "utf-8") as f: + with open(cfg_path, encoding = "utf-8-sig") as f: cfg = json.load(f) except Exception: return None @@ -3046,6 +3047,7 @@ class LlamaCppBackend: [bin_path, "--help"], capture_output = True, text = True, + encoding = "utf-8", errors = "replace", timeout = 10, check = False, @@ -3618,6 +3620,8 @@ class LlamaCppBackend: ], capture_output = True, text = True, + encoding = "utf-8", + errors = "replace", timeout = 10, env = child_env_without_native_path_secret(), **_windows_hidden_subprocess_kwargs(), @@ -3732,7 +3736,7 @@ class LlamaCppBackend: encoding = "utf-8", errors = "replace", timeout = 15, - env = env, + env = utf8_child_env(env), **_windows_hidden_subprocess_kwargs(), ) if result.returncode != 0: @@ -5482,7 +5486,9 @@ class LlamaCppBackend: stdout = subprocess.PIPE, stderr = subprocess.STDOUT, text = True, - env = env, + encoding = "utf-8", + errors = "replace", + env = utf8_child_env(env), **_windows_hidden_subprocess_kwargs(), **_child_popen_kwargs(), ) @@ -6696,6 +6702,8 @@ class LlamaCppBackend: stdout = subprocess.PIPE, stderr = subprocess.STDOUT, text = True, + encoding = "utf-8", + errors = "replace", env = env, **_windows_hidden_subprocess_kwargs(), **_child_popen_kwargs(), @@ -8712,6 +8720,8 @@ class LlamaCppBackend: stdout = subprocess.PIPE, stderr = subprocess.STDOUT, text = True, + encoding = "utf-8", + errors = "replace", env = env, **_windows_hidden_subprocess_kwargs(), **_child_popen_kwargs(), @@ -10214,6 +10224,8 @@ class LlamaCppBackend: ["pgrep", "-a", "-f", "llama-server"], capture_output = True, text = True, + encoding = "utf-8", + errors = "replace", timeout = 5, env = child_env_without_native_path_secret(), ) diff --git a/studio/backend/core/inference/worker.py b/studio/backend/core/inference/worker.py index 3f32b3bd57..f208183300 100644 --- a/studio/backend/core/inference/worker.py +++ b/studio/backend/core/inference/worker.py @@ -151,7 +151,7 @@ def _resolve_lora_4bit(mc, load_in_4bit: bool) -> bool: import json try: - with open(adapter_cfg_path, encoding = "utf-8") as f: + with open(adapter_cfg_path, encoding = "utf-8-sig") as f: adapter_cfg = json.load(f) training_method = adapter_cfg.get("unsloth_training_method") if training_method == "lora" and load_in_4bit: @@ -963,7 +963,7 @@ def run_inference_process( if _local_adapter_cfg.is_file(): try: _lora_base = ( - _json.loads(_local_adapter_cfg.read_text(encoding = "utf-8")).get( + _json.loads(_local_adapter_cfg.read_text(encoding = "utf-8-sig")).get( "base_model_name_or_path" ) or None diff --git a/studio/backend/core/rag/embed_llama_server.py b/studio/backend/core/rag/embed_llama_server.py index facd989b27..b3ac62e520 100644 --- a/studio/backend/core/rag/embed_llama_server.py +++ b/studio/backend/core/rag/embed_llama_server.py @@ -103,6 +103,8 @@ class LlamaServerBackend: [binary, "--help"], capture_output = True, text = True, + encoding = "utf-8", + errors = "replace", timeout = 30, **windows_hidden_subprocess_kwargs(), ) @@ -331,6 +333,8 @@ class LlamaServerBackend: stdout = subprocess.PIPE, stderr = subprocess.STDOUT, text = True, + encoding = "utf-8", + errors = "replace", env = env, **windows_hidden_subprocess_kwargs(), **child_popen_kwargs(), diff --git a/studio/backend/core/rag/embeddings.py b/studio/backend/core/rag/embeddings.py index c86c0d3c51..95b8a866b2 100644 --- a/studio/backend/core/rag/embeddings.py +++ b/studio/backend/core/rag/embeddings.py @@ -100,7 +100,7 @@ def _st_module_subdirs(name: str, token: str | None) -> tuple[str, ...]: path = Path(normalize_path(name)).expanduser() / "modules.json" if not path.is_file(): return () - data = json.loads(path.read_text(encoding = "utf-8")) + data = json.loads(path.read_text(encoding = "utf-8-sig")) else: from huggingface_hub import hf_hub_download from huggingface_hub.utils import EntryNotFoundError @@ -115,7 +115,7 @@ def _st_module_subdirs(name: str, token: str | None) -> tuple[str, ...]: ) except EntryNotFoundError: return () - data = json.loads(open(local, encoding = "utf-8").read()) + data = json.loads(open(local, encoding = "utf-8-sig").read()) subdirs = [] for module in data or (): sub = str((module or {}).get("path", "")).strip().strip("/") diff --git a/studio/backend/core/training/worker.py b/studio/backend/core/training/worker.py index baf6329dae..b5fb5d224e 100644 --- a/studio/backend/core/training/worker.py +++ b/studio/backend/core/training/worker.py @@ -43,6 +43,7 @@ if sys.platform.startswith("linux") and "HSA_ENABLE_DXG_DETECTION" not in os.env pass logger = get_logger(__name__) +from utils.child_stdio import utf8_child_env from utils.hardware import apply_gpu_ids from utils.training_runs import build_default_output_dir_name from utils.wheel_utils import ( @@ -385,6 +386,10 @@ def _install_package_wheel_first( "stdout": _sp.PIPE, "stderr": _sp.STDOUT, "text": True, + "encoding": "utf-8", + "errors": "replace", + # Make the Python child emit the UTF-8 we decode above. + "env": utf8_child_env(), } if is_hip: _run_kwargs["timeout"] = 1800 @@ -606,6 +611,9 @@ def _ensure_flash_linear_attention_unconditional(event_queue: Any) -> bool: stdout = _sp.PIPE, stderr = _sp.STDOUT, text = True, + encoding = "utf-8", + errors = "replace", + env = utf8_child_env(), timeout = _TILELANG_INSTALL_TIMEOUT_S, ) except _sp.TimeoutExpired: @@ -849,6 +857,9 @@ def _run_pip(cmd: list[str], event_queue: Any, label: str) -> bool: stdout = _sp.PIPE, stderr = _sp.STDOUT, text = True, + encoding = "utf-8", + errors = "replace", + env = utf8_child_env(), timeout = _TILELANG_INSTALL_TIMEOUT_S, ) except _sp.TimeoutExpired: diff --git a/studio/backend/hub/services/models/ollama.py b/studio/backend/hub/services/models/ollama.py index 56275c22a9..da30f7e98c 100644 --- a/studio/backend/hub/services/models/ollama.py +++ b/studio/backend/hub/services/models/ollama.py @@ -215,7 +215,7 @@ def _ollama_model_info_from_manifest( return None try: - manifest = json.loads(tag_file.read_text(encoding = "utf-8")) + manifest = json.loads(tag_file.read_text(encoding = "utf-8-sig")) except (json.JSONDecodeError, OSError, UnicodeDecodeError) as e: logger.debug("Skipping unreadable/invalid Ollama manifest %s: %s", tag_file, e) return None @@ -228,7 +228,7 @@ def _ollama_model_info_from_manifest( config_blob = _ollama_blob_path(blobs_dir, config_digest) if config_blob is not None and _safe_is_file(config_blob): try: - cfg = json.loads(config_blob.read_text(encoding = "utf-8")) + cfg = json.loads(config_blob.read_text(encoding = "utf-8-sig")) model_type = cfg.get("model_type", "") file_type = cfg.get("file_type", "") except (json.JSONDecodeError, OSError, UnicodeDecodeError) as e: diff --git a/studio/backend/hub/utils/download_registry.py b/studio/backend/hub/utils/download_registry.py index 39c27208b1..760ef6b01c 100644 --- a/studio/backend/hub/utils/download_registry.py +++ b/studio/backend/hub/utils/download_registry.py @@ -464,6 +464,8 @@ def _read_marker_value(marker: Path) -> Optional[str]: return None value = marker.read_text(encoding = "utf-8").strip() except (OSError, UnicodeDecodeError): + # UnicodeDecodeError is a ValueError, so it would escape and abort + # prepare_cache_for_transport. An unknown value just purges and restarts. return None return value if value in VALID_TRANSPORTS else None diff --git a/studio/backend/loggers/config.py b/studio/backend/loggers/config.py index 688d3c7ebe..57cf7cecd6 100644 --- a/studio/backend/loggers/config.py +++ b/studio/backend/loggers/config.py @@ -42,8 +42,12 @@ class LogConfig: log_level_name = os.getenv("LOG_LEVEL", "INFO").upper() log_level = getattr(logging, log_level_name, logging.INFO) - if sys.platform == "win32": - for stream in (sys.stdout, sys.stderr): + # Non-ASCII on a non-UTF-8 stream raises UnicodeEncodeError (Windows, + # LANG=C), so key off the stream, not the platform. + for stream in (sys.stdout, sys.stderr): + if getattr(stream, "encoding", "") and not str(stream.encoding).lower().replace( + "-", "" + ).startswith("utf8"): if hasattr(stream, "reconfigure"): try: stream.reconfigure(encoding = "utf-8", errors = "replace") diff --git a/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/state_store.py b/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/state_store.py index b4c226136b..b059fad7ff 100644 --- a/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/state_store.py +++ b/studio/backend/plugins/data-designer-github-repo-seed/src/data_designer_github_repo_seed/scraper_impl/state_store.py @@ -6,10 +6,93 @@ from __future__ import annotations import json +import locale import os import threading from pathlib import Path -from typing import Any, Dict +from typing import Any, Dict, NamedTuple + + +def _locale_encoding() -> str: + """The codepage a pre-UTF-8 release here would have written, or "". + + Empty on a UTF-8 host, where there is no codepage to attribute the file to. + """ + try: + preferred = locale.getencoding() + except AttributeError: # Python < 3.11 + preferred = locale.getpreferredencoding(False) + if preferred.lower().replace("-", "").replace("_", "") == "utf8": + return "" + return preferred + + +# Trail bytes can land on JSON punctuation, so a single-byte fallback misreads these. +_DOUBLE_BYTE_ENCODINGS = ("cp932", "cp936", "cp949", "cp950") + + +def _parse(raw: bytes, encoding: str) -> Any: + """Parse one JSON document under *encoding*, or None if it does not. + + RecursionError is a RuntimeError, so nesting json.loads will not descend is + the one parse failure the other three miss. Both callers run this outside + any further handler, so it has to answer None here or a single damaged + record aborts the scraper at startup instead of being skipped. + """ + try: + return json.loads(raw.decode(encoding)) + except (UnicodeDecodeError, LookupError, ValueError, RecursionError): + return None + + +class _Reading(NamedTuple): + as_utf8: Any + as_legacy: Any + + +def _read_line(raw: bytes, codepage: str) -> _Reading: + """Read one line as UTF-8 and as a codepage, for dedup keys only. + + Requiring valid JSON, not merely a successful decode, is what separates a + genuine legacy record from a half-written UTF-8 one: a torn multibyte + character decodes under cp1252 but leaves the JSON unterminated. Some byte + strings parse both ways, e.g. cp1251 ``Р°`` is ``D0 B0``, which is also + UTF-8 ``а``. + + The codepage reading is never authoritative, because the file's own encoding + cannot be recovered from its bytes. Reading a cp1251 shard on a cp1252 + machine turns ``Привет`` into ``Ïðèâåò`` and every byte of it decodes + cleanly, so a successful decode proves nothing about who wrote it. It is + used only to recover the dedup keys, which are ASCII ids and come back the + same under any of these, so the first reading that parses will do. + + That is also why several are tried. latin-1 alone mangles the double-byte + codepages: cp932 ``表`` is ``95 5C``, and latin-1 turns the trail byte into + a JSON backslash, so the record fails to parse and its id is forgotten. + """ + as_utf8 = _parse(raw, "utf-8") + # A record that reads as UTF-8 needs no second reading: re-parsing cost 2.8x on a + # 76 MB shard, and these reach gigabytes. Only a dict, since key lookup falls + # through to the codepage when UTF-8 yields none. + if isinstance(as_utf8, dict): + return _Reading(as_utf8, None) + for encoding in (codepage, "latin-1", *_DOUBLE_BYTE_ENCODINGS): + if not encoding: + continue + as_legacy = _parse(raw, encoding) + if as_legacy is not None: + return _Reading(as_utf8, as_legacy) + return _Reading(as_utf8, None) + + +class _Scan(NamedTuple): + """What a pass over an existing shard established about it.""" + + legacy: bool # enough evidence to trust the codepage reading's keys + readable: bool + saw_non_ascii: bool # some line's meaning depends on the encoding + utf8_keys: set # keys from lines UTF-8 could read + legacy_keys: set # keys only the codepage reading yields class StateStore: @@ -18,12 +101,19 @@ class StateStore: self.path.parent.mkdir(parents = True, exist_ok = True) self._lock = threading.Lock() self._data: Dict[str, Any] = {} + # Read whole, and UTF-8 only unlike the shards below: a checkpoint holds + # nothing but base64 cursors and booleans, so a codepage retry could only ever + # add non-ASCII. That would resume on a mojibaked cursor, which GitHub rejects + # with INVALID_CURSOR_ARGUMENTS, and the empty page it returns marks the stream + # done and skips the rest for good. Dropping a damaged checkpoint re-scrapes + # from the first page, which the writers dedup. if self.path.exists(): try: - with self.path.open(encoding = "utf-8") as f: - self._data = json.load(f) - except Exception: - self._data = {} + raw = self.path.read_bytes() + except OSError: + raw = b"" + data = _parse(raw, "utf-8") + self._data = data if isinstance(data, dict) else {} def get( self, @@ -63,24 +153,83 @@ class JsonlWriter: self.path = Path(path) self.path.parent.mkdir(parents = True, exist_ok = True) self._lock = threading.Lock() - self._fh = self.path.open("a", buffering = 1, encoding = "utf-8") self._count_seen_keys: set[str] = set() - # Preload seen keys for dedup across resumes + self._codepage = _locale_encoding() + self._ensure_ascii = False + encoding = "utf-8" if self.path.exists() and self.path.stat().st_size > 0: - try: - # No guess is safe for a file an older build wrote in the - # operator's locale, so read past whatever will not decode. - with self.path.open(encoding = "utf-8", errors = "replace") as f: - for line in f: - try: - obj = json.loads(line) - k = self._key(obj) - if k is not None: - self._count_seen_keys.add(k) - except Exception: - pass - except Exception: - pass + scan = self._scan_existing() + self._count_seen_keys = scan.utf8_keys + if scan.legacy: + self._count_seen_keys |= scan.legacy_keys + if scan.saw_non_ascii or not scan.readable: + # Never convert: the writing encoding is unrecoverable and guessing + # mojibakes the records. Pure ASCII appends store identically under + # every codepage, and json.loads turns the \uXXXX escapes back. + encoding = "ascii" + self._ensure_ascii = True + self._fh = self.path.open("a", buffering = 1, encoding = encoding, errors = "strict") + + def _scan_existing(self) -> _Scan: + """Read the shard once to recover dedup keys and judge its encoding. + + Line by line: these shards reach gigabytes on a large scrape, so neither + the bytes nor the decoded text are held whole. + + The verdict weighs the whole file. Each line with non-ASCII bytes votes: + one that parses only under the codepage is evidence of a legacy shard, + one that parses as UTF-8 is evidence against, since arbitrary codepage + text almost never forms valid multibyte UTF-8. A single corrupt byte in + a healthy shard therefore cannot outvote the records around it, and a + genuinely legacy shard has a legacy vote on every line that carries an + umlaut. + + More than one such line is required, because a single one is genuinely + undecidable: a legacy record holding one accented character and an ASCII + record holding one stray byte are the same shape. Reading it as damage + risks a duplicate; reading it as legacy marks an unreadable record seen + and blocks the retry that would replace it, losing it for good. Only one + of those is recoverable. + + The verdict only picks which reading supplies the dedup keys. The file + itself is never rewritten either way, so a wrong answer costs at most a + duplicate, never a corrupted record. + """ + legacy_votes = 0 + utf8_votes = 0 + saw_non_ascii = False + utf8_keys: set[str] = set() + legacy_keys: set[str] = set() + try: + with self.path.open("rb") as handle: + for raw in handle: + line = raw.strip() + reading = _read_line(line, self._codepage) + # ASCII reads the same everywhere: no vote, no constraint. + if not line.isascii(): + saw_non_ascii = True + if reading.as_utf8 is None and reading.as_legacy is not None: + legacy_votes += 1 + elif reading.as_utf8 is not None: + utf8_votes += 1 + # Kept apart so a damaged line does not block its own retry. + if isinstance(reading.as_utf8, dict): + key = self._key(reading.as_utf8) + if key is not None: + utf8_keys.add(key) + elif isinstance(reading.as_legacy, dict): + key = self._key(reading.as_legacy) + if key is not None: + legacy_keys.add(key) + except OSError: + return _Scan(False, False, False, utf8_keys, legacy_keys) + return _Scan( + legacy_votes > 1 and legacy_votes > utf8_votes, + True, + saw_non_ascii, + utf8_keys, + legacy_keys, + ) def _key(self, obj: dict) -> str | None: for k in ("id", "node_id", "number", "sha", "url"): @@ -99,7 +248,7 @@ class JsonlWriter: return False if k is not None: self._count_seen_keys.add(k) - self._fh.write(json.dumps(obj, default = str, ensure_ascii = False)) + self._fh.write(json.dumps(obj, default = str, ensure_ascii = self._ensure_ascii)) self._fh.write("\n") self._fh.flush() return True diff --git a/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/impl.py b/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/impl.py index ce0c88e5bf..825b050e07 100644 --- a/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/impl.py +++ b/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/impl.py @@ -30,6 +30,8 @@ class UnstructuredSeedReader(SeedReader[UnstructuredSeedSource]): meta = json_mod.loads(meta_path.read_text(encoding = "utf-8")) orig_name = meta.get("original_filename", path_obj.name) except (json_mod.JSONDecodeError, OSError, UnicodeDecodeError): + # Undecodable metadata is as malformed as invalid JSON, so + # fall back to the file's own name rather than abort the seed. pass file_entries.append((path_obj, orig_name)) diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index d0a2d97f74..20a5af1409 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -4434,7 +4434,7 @@ def _effective_load_in_4bit(config: ModelConfig, requested: bool) -> bool: if not adapter_cfg_path.exists(): return load_in_4bit try: - with open(adapter_cfg_path, encoding = "utf-8") as f: + with open(adapter_cfg_path, encoding = "utf-8-sig") as f: adapter_cfg = json.load(f) if not isinstance(adapter_cfg, dict): # malformed -> keep requested return load_in_4bit diff --git a/studio/backend/routes/models.py b/studio/backend/routes/models.py index 96c5b96d73..6e587c18e8 100644 --- a/studio/backend/routes/models.py +++ b/studio/backend/routes/models.py @@ -722,7 +722,7 @@ def _scan_ollama_dir(ollama_dir: Path, limit: Optional[int] = None) -> List[Loca stem_hash = hashlib.sha256(manifest_key.encode()).hexdigest()[:10] try: - manifest = json.loads(tag_file.read_text(encoding = "utf-8")) + manifest = json.loads(tag_file.read_text(encoding = "utf-8-sig")) except (json.JSONDecodeError, OSError, UnicodeDecodeError) as e: logger.debug( "Skipping unreadable/invalid Ollama manifest %s: %s", @@ -738,7 +738,7 @@ def _scan_ollama_dir(ollama_dir: Path, limit: Optional[int] = None) -> List[Loca config_blob = blobs_dir / config_digest.replace(":", "-") if config_blob.is_file(): try: - cfg = json.loads(config_blob.read_text(encoding = "utf-8")) + cfg = json.loads(config_blob.read_text(encoding = "utf-8-sig")) model_type = cfg.get("model_type", "") file_type = cfg.get("file_type", "") except (json.JSONDecodeError, OSError, UnicodeDecodeError) as e: @@ -1042,7 +1042,7 @@ def _dir_has_downloaded_model(directory: Path, max_entries: int = 4000) -> bool: if not m.is_file(): continue try: - manifest = json.loads(m.read_text(encoding = "utf-8")) + manifest = json.loads(m.read_text(encoding = "utf-8-sig")) except (json.JSONDecodeError, OSError, ValueError): continue for layer in manifest.get("layers") or []: @@ -3360,6 +3360,8 @@ def _wsl_reveal_in_explorer(path: Path) -> bool: ["wslpath", "-w", str(path)], capture_output = True, text = True, + encoding = "utf-8", + errors = "replace", check = True, timeout = 10, ).stdout.strip() diff --git a/studio/backend/run.py b/studio/backend/run.py index 08d1c5299e..ef372e004e 100644 --- a/studio/backend/run.py +++ b/studio/backend/run.py @@ -786,6 +786,8 @@ def _remove_pid_file(): stored = _PID_FILE.read_text(encoding = "utf-8").strip() if stored == str(os.getpid()): _PID_FILE.unlink(missing_ok = True) + # Runs first in _graceful_shutdown: a corrupt PID file raising here would + # abandon the children the rest of that function exists to kill. except (OSError, UnicodeDecodeError): pass diff --git a/studio/backend/tests/test_chat_text_encoding.py b/studio/backend/tests/test_chat_text_encoding.py new file mode 100644 index 0000000000..64860dab1a --- /dev/null +++ b/studio/backend/tests/test_chat_text_encoding.py @@ -0,0 +1,195 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Model text stays intact when it carries non-ASCII. + +``open()`` and ``Path.read_text()`` fall back to ``locale.getencoding()`` when +no ``encoding`` is passed. On Windows that is the ANSI codepage, not UTF-8, so +a chat template or model config holding ``ä ö ü → 世`` mojibakes or raises +``UnicodeDecodeError``. These files are UTF-8, so the reads must say so. + +Each fixture writes raw UTF-8 (``ensure_ascii = False``), matching what +Hugging Face actually ships, rather than ASCII ``\\uXXXX`` escapes. +""" + +from __future__ import annotations + +import json +import subprocess +import sys +import textwrap +from pathlib import Path + + +BACKEND_ROOT = Path(__file__).resolve().parent.parent + + +def test_config_json_round_trips_non_ascii(tmp_path: Path) -> None: + from utils import transformers_version + + name = "Modell für Grüße 世界" + (tmp_path / "config.json").write_text( + json.dumps({"model_type": "llama", "_name_or_path": name}, ensure_ascii = False), + encoding = "utf-8", + ) + transformers_version._config_json_cache.clear() + + cfg = transformers_version._load_config_json(str(tmp_path)) + + assert cfg is not None + assert cfg["_name_or_path"] == name + + +def test_tokenizer_config_round_trips_non_ascii_chat_template(tmp_path: Path) -> None: + """Chat templates commonly hold ``→`` and smart quotes, which cp1252 mangles.""" + from utils import transformers_version + + template = "{{ '→ Grüße 世界' }}" + (tmp_path / "tokenizer_config.json").write_text( + json.dumps( + {"tokenizer_class": "TokenizersBackend", "chat_template": template}, + ensure_ascii = False, + ), + encoding = "utf-8", + ) + transformers_version._tokenizer_class_cache.clear() + + assert transformers_version._check_tokenizer_config_needs_v5(str(tmp_path)) is True + + +def test_config_json_survives_a_utf8_bom(tmp_path: Path) -> None: + """Notepad wrote "UTF-8 with BOM" by default for years, so hand-edited + configs on Windows carry one. Plain utf-8 keeps the BOM and json.load then + fails on it; utf-8-sig strips it and is identical otherwise.""" + from utils import transformers_version + + name = "Grüße 世界" + (tmp_path / "config.json").write_text( + json.dumps({"model_type": "llama", "_name_or_path": name}, ensure_ascii = False), + encoding = "utf-8-sig", + ) + transformers_version._config_json_cache.clear() + + cfg = transformers_version._load_config_json(str(tmp_path)) + + assert cfg is not None + assert cfg["_name_or_path"] == name + + +def test_remote_code_scan_reads_non_ascii_sources(tmp_path: Path) -> None: + """A German Windows profile also puts umlauts in the model sources scanned.""" + from utils.security import remote_code_scan + + source = "# Grüße über Öl\nVALUE = '世界'\n" + # newline = "" pins the bytes on disk, so Windows line end translation cannot make the + # read back differ by \r. open() because Path.write_text() only grew newline in 3.10. + with open( + tmp_path / "modeling_custom.py", + "w", + encoding = "utf-8", + newline = "", + ) as handle: + handle.write(source) + + files = remote_code_scan.repo_remote_code_files(str(tmp_path)) + + assert files["modeling_custom.py"] == source + + +def test_model_config_reads_do_not_rely_on_the_locale_encoding(tmp_path: Path) -> None: + """The reads above pass anywhere the locale is already UTF-8, which hides + the Windows bug on Linux and macOS. ``-X warn_default_encoding`` makes + CPython flag any text I/O that falls back to the locale, so this fails on + every platform if an ``encoding`` argument goes missing again.""" + # The readers swallow exceptions, so record the warnings instead of raising. + script = textwrap.dedent( + f""" + import sys, warnings + sys.path.insert(0, {str(BACKEND_ROOT)!r}) + from utils import transformers_version + + target = {str(tmp_path)!r} + with warnings.catch_warnings(record = True) as caught: + warnings.simplefilter("always") + transformers_version._config_json_cache.clear() + transformers_version._tokenizer_class_cache.clear() + assert transformers_version._load_config_json(target) is not None + assert transformers_version._check_tokenizer_config_needs_v5(target) is True + + missing = [str(w.message) for w in caught if w.category is EncodingWarning] + if missing: + sys.exit("text I/O fell back to the locale encoding: " + "; ".join(missing)) + """ + ) + for name, payload in ( + ("config.json", {"model_type": "llama", "_name_or_path": "Grüße"}), + ("tokenizer_config.json", {"tokenizer_class": "TokenizersBackend"}), + ): + (tmp_path / name).write_text(json.dumps(payload, ensure_ascii = False), encoding = "utf-8") + + result = subprocess.run( + [sys.executable, "-X", "warn_default_encoding", "-c", script], + capture_output = True, + text = True, + encoding = "utf-8", + errors = "replace", + timeout = 120, + ) + + assert result.returncode == 0, result.stderr + + +def test_utf8_child_env_round_trips_non_ascii(tmp_path: Path) -> None: + """A Python child encodes stdout with its locale unless told otherwise, so + reading its pipe as utf-8 needs the child told to emit utf-8.""" + from utils.child_stdio import utf8_child_env + + payload = "Grüße über Öl → 世界" + child = tmp_path / "child.py" + child.write_text("import sys\nsys.stdout.write(" + repr(payload) + ")\n", encoding = "utf-8") + + env = utf8_child_env() + assert env["PYTHONIOENCODING"] == "utf-8" + + proc = subprocess.run( + [sys.executable, str(child)], + capture_output = True, + text = True, + encoding = "utf-8", + errors = "replace", + env = env, + timeout = 120, + ) + + assert proc.returncode == 0, proc.stderr + assert proc.stdout == payload + + +def test_python_children_are_told_to_emit_utf8() -> None: + """Any child we decode as utf-8 must also be told to write utf-8, or a + cp1252 console silently mangles what it prints.""" + import ast + + offenders: list[str] = [] + for path in sorted(BACKEND_ROOT.rglob("*.py")): + parts = path.relative_to(BACKEND_ROOT).parts + if any(p in ("tests", "node_modules", "plugins", "__pycache__") for p in parts): + continue + source = path.read_text(encoding = "utf-8") + for node in ast.walk(ast.parse(source, filename = str(path))): + if not isinstance(node, ast.Call): + continue + func = node.func + if not (isinstance(func, ast.Attribute) and func.attr in ("run", "Popen")): + continue + segment = ast.get_source_segment(source, node) or "" + if "sys.executable" not in segment or 'encoding = "utf-8"' not in segment: + continue + if "utf8_child_env" in segment or "PYTHONIOENCODING" in segment: + continue + offenders.append(f"{path.name}:{node.lineno}") + + assert not offenders, ( + "these spawn a Python child and decode it as utf-8 without setting the " + "child's own stdio encoding; wrap env in utf8_child_env():\n " + "\n ".join(offenders) + ) diff --git a/studio/backend/tests/test_rocm_multi_gpu_vram_system_wide.py b/studio/backend/tests/test_rocm_multi_gpu_vram_system_wide.py index bdafdeae9b..db89b02003 100644 --- a/studio/backend/tests/test_rocm_multi_gpu_vram_system_wide.py +++ b/studio/backend/tests/test_rocm_multi_gpu_vram_system_wide.py @@ -45,8 +45,20 @@ def _build_structlog_stub(): _maybe_stub("loggers", _build_loggers_stub) _maybe_stub("structlog", _build_structlog_stub) +import pytest + import utils.hardware.hardware as hw # noqa: E402 +# The DRM/KFD readers below are Linux-only in production: _rocm_linux_amdgpu_cards and +# _rocm_linux_sysfs_vram_by_pci_gb return early unless platform.system() is "Linux", and +# _rocm_kfd_gpu_pci_ids only ever globs /sys/class/kfd. Their fake sysfs tree needs PCI +# addresses like "0000:00:02.0" as directory names and POSIX separators in the paths the +# readers match; Windows permits neither, so the tree cannot be represented there. +linux_only = pytest.mark.skipif( + not sys.platform.startswith("linux"), + reason = "covers Linux-only DRM/KFD sysfs parsing driven by a fake /sys tree", +) + def _device( index, @@ -99,6 +111,7 @@ def _fake_drm(tmp_path, monkeypatch, cards): return card_paths +@linux_only def test_linux_vram_keyed_by_pci_excludes_foreign_adapters(monkeypatch, tmp_path): # Foreign (non-amdgpu) adapters contribute no entry, so they cannot shift ordinals. monkeypatch.setattr(hw.platform, "system", lambda: "Linux") @@ -117,6 +130,7 @@ def test_linux_vram_keyed_by_pci_excludes_foreign_adapters(monkeypatch, tmp_path } +@linux_only def test_linux_vram_omits_bad_cards_without_shifting(monkeypatch, tmp_path): # A zero-total card has no entry; identity keying means its absence renumbers nothing. monkeypatch.setattr(hw.platform, "system", lambda: "Linux") @@ -131,6 +145,7 @@ def test_linux_vram_omits_bad_cards_without_shifting(monkeypatch, tmp_path): assert hw._rocm_linux_sysfs_vram_by_pci_gb() == {"0000:41:00.0": (2.0, 16.0)} +@linux_only def test_linux_vram_omits_amd_card_without_vram_files(monkeypatch, tmp_path): # An APU with no mem_info_vram_* files has no entry; the discrete card keeps its address. monkeypatch.setattr(hw.platform, "system", lambda: "Linux") @@ -174,6 +189,7 @@ def _fake_kfd(tmp_path, monkeypatch, nodes): return node_paths +@linux_only def test_kfd_lists_gpu_nodes_in_device_order(monkeypatch, tmp_path): # The CPU node (simd_count 0) takes no ordinal; GPU nodes in node-id order are HIP's order. monkeypatch.setattr(hw.platform, "system", lambda: "Linux") @@ -189,12 +205,14 @@ def test_kfd_lists_gpu_nodes_in_device_order(monkeypatch, tmp_path): assert hw._rocm_kfd_gpu_pci_ids() == ["0000:03:00.0", "0000:41:00.0"] +@linux_only def test_kfd_decodes_domain_device_and_function(monkeypatch, tmp_path): monkeypatch.setattr(hw.platform, "system", lambda: "Linux") _fake_kfd(tmp_path, monkeypatch, [(1, 64, (0xC1 << 8) | (0x1F << 3) | 5, 0x1234, _AMD)]) assert hw._rocm_kfd_gpu_pci_ids() == ["1234:c1:1f.5"] +@linux_only def test_kfd_skips_non_amd_gpu_nodes(monkeypatch, tmp_path): # An NVIDIA KFD node is not a HIP device: it must take no ordinal, else it # shifts every AMD GPU and ROCm device 1 resolves to AMD GPU 0. @@ -212,6 +230,7 @@ def test_kfd_skips_non_amd_gpu_nodes(monkeypatch, tmp_path): assert hw._rocm_kfd_gpu_pci_ids() == ["0000:03:00.0", "0000:41:00.0"] +@linux_only def test_kfd_fails_closed_when_a_gpu_has_no_location(monkeypatch, tmp_path): # Dropping an unplaceable AMD GPU shifts later ordinals; fail closed for the whole map. monkeypatch.setattr(hw.platform, "system", lambda: "Linux") @@ -226,6 +245,7 @@ def test_kfd_fails_closed_when_a_gpu_has_no_location(monkeypatch, tmp_path): assert hw._rocm_kfd_gpu_pci_ids() == [] +@linux_only def test_kfd_fails_closed_when_a_node_is_unreadable(monkeypatch, tmp_path): # An unreadable node could be a GPU; assuming otherwise would shift ordinals. monkeypatch.setattr(hw.platform, "system", lambda: "Linux") @@ -241,6 +261,23 @@ def test_kfd_fails_closed_when_a_node_is_unreadable(monkeypatch, tmp_path): assert hw._rocm_kfd_gpu_pci_ids() == [] +@linux_only +def test_kfd_fails_closed_when_a_node_does_not_decode(monkeypatch, tmp_path): + # UnicodeDecodeError is a ValueError, so it slips past `except OSError` and + # would shift every later HIP ordinal. + monkeypatch.setattr(hw.platform, "system", lambda: "Linux") + paths = _fake_kfd( + tmp_path, + monkeypatch, + [ + (1, 304, (0x03 << 8) | 0, 0, _AMD), + (2, 304, (0x41 << 8) | 0, 0, _AMD), + ], + ) + (Path(paths[0]) / "properties").write_bytes(b"simd_count 304\nvendor_id \x80\xff\n") + assert hw._rocm_kfd_gpu_pci_ids() == [] + + def test_kfd_absent_yields_no_device_order(monkeypatch): monkeypatch.setattr(hw.glob, "glob", lambda pattern: []) assert hw._rocm_kfd_gpu_pci_ids() == [] @@ -422,6 +459,10 @@ def test_visible_utilization_rocm_fallback_overlays(monkeypatch): ): monkeypatch.delenv(_var, raising = False) monkeypatch.setattr(hw, "IS_ROCM", True) + # No AMD adapter data on this host. On Windows this branch runs ahead of the torch + # fallback under test, and probing it imports torch, which the CI runner does not + # install. Off Windows the real function is never reached, so this changes nothing. + monkeypatch.setattr(hw, "_rocm_windows_per_device_vram", lambda ids: []) monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.CUDA) monkeypatch.setattr(hw, "_smi_query", lambda *a, **k: None) # amd-smi unavailable monkeypatch.setattr( @@ -450,6 +491,10 @@ def test_visible_utilization_rocm_fallback_overlays(monkeypatch): def test_visible_utilization_relative_index_skips_overlay(monkeypatch): # UUID/MIG mask gives relative indices; the overlay matches physical index, so it must not run. monkeypatch.setattr(hw, "IS_ROCM", True) + # No AMD adapter data on this host. On Windows this branch runs ahead of the torch + # fallback under test, and probing it imports torch, which the CI runner does not + # install. Off Windows the real function is never reached, so this changes nothing. + monkeypatch.setattr(hw, "_rocm_windows_per_device_vram", lambda ids: []) monkeypatch.setattr(hw, "get_device", lambda: hw.DeviceType.CUDA) monkeypatch.setattr(hw, "_smi_query", lambda *a, **k: None) monkeypatch.setattr( diff --git a/studio/backend/tests/test_text_io_encoding.py b/studio/backend/tests/test_text_io_encoding.py new file mode 100644 index 0000000000..7eae3c7fef --- /dev/null +++ b/studio/backend/tests/test_text_io_encoding.py @@ -0,0 +1,809 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Text I/O must name its encoding, or Windows silently uses the ANSI codepage. + +``open()``, ``Path.read_text()`` and ``subprocess(text = True)`` fall back to +``locale.getencoding()`` when no ``encoding`` is passed. On Windows that is +cp1252 (or cp932, cp1251, ... by system locale), not UTF-8, so a chat template, +model config or path containing ``ä ö ü → 世`` mojibakes or raises +``UnicodeDecodeError`` mid-load. Studio's files are UTF-8, so say so. +""" + +from __future__ import annotations + +import ast +import importlib.util +import json +import os +from pathlib import Path +from types import SimpleNamespace + +import pytest + + +BACKEND_ROOT = Path(__file__).resolve().parent.parent + +# Not runtime source. Shipped plugins under plugins/*/src are, so only builds are skipped. +_SKIPPED_DIRS = ("node_modules", "build", "tests", "__pycache__") + +# Path.open()'s signature is what tells it apart from other libraries' open(), +# e.g. fitz.open(stream=...) and av.open(..., metadata_errors=...). +_FILE_MODE_CHARS = set("rwxabt+") +_PATH_OPEN_ARGS = ("mode", "buffering", "encoding", "errors", "newline") +_PATH_OPEN_KWARGS = set(_PATH_OPEN_ARGS) +_PATH_OPEN_ENCODING_ARG = _PATH_OPEN_ARGS.index("encoding") + +_SUBPROCESS_CALLS = {"run", "Popen", "check_output", "check_call", "call"} + +# open(file, mode, buffering, encoding, ...), and os.fdopen forwards the same +# signature with a descriptor in place of the path. +_OPEN_ENCODING_ARG = 3 + + +def _studio_sources() -> list[Path]: + return [ + path + for path in sorted(BACKEND_ROOT.rglob("*.py")) + if not any(part in _SKIPPED_DIRS for part in path.relative_to(BACKEND_ROOT).parts) + ] + + +def _has_keyword(node: ast.Call, name: str) -> bool: + return any(keyword.arg == name for keyword in node.keywords) + + +def _mode_is_binary(node: ast.Call) -> bool: + mode: str | None = None + if len(node.args) >= 2 and isinstance(node.args[1], ast.Constant): + value = node.args[1].value + mode = value if isinstance(value, str) else None + for keyword in node.keywords: + if keyword.arg == "mode" and isinstance(keyword.value, ast.Constant): + value = keyword.value.value + if isinstance(value, str): + mode = value + return bool(mode and "b" in mode) + + +def _open_has_encoding(node: ast.Call) -> bool: + """open()/os.fdopen() also take encoding positionally: open(p, "w", 1, "utf-8").""" + return _has_keyword(node, "encoding") or len(node.args) > _OPEN_ENCODING_ARG + + +def _path_open_mode(node: ast.Call) -> str | None: + if node.args and isinstance(node.args[0], ast.Constant): + value = node.args[0].value + if isinstance(value, str): + return value + for keyword in node.keywords: + if keyword.arg == "mode" and isinstance(keyword.value, ast.Constant): + value = keyword.value.value + if isinstance(value, str): + return value + return None + + +def _is_path_open(node: ast.Call) -> bool: + """True only for calls matching ``Path.open``'s signature.""" + if len(node.args) > len(_PATH_OPEN_ARGS): + return False + if any(k.arg not in _PATH_OPEN_KWARGS for k in node.keywords): + return False + mode = _path_open_mode(node) + if mode is not None: + return bool(mode) and set(mode) <= _FILE_MODE_CHARS + return not node.args + + +def _path_open_has_encoding(node: ast.Call) -> bool: + """Path.open() also takes encoding positionally: open("w", 1, "utf-8").""" + return _has_keyword(node, "encoding") or len(node.args) > _PATH_OPEN_ENCODING_ARG + + +def _call_name(node: ast.Call) -> str | None: + func = node.func + if isinstance(func, ast.Name): + return func.id + if isinstance(func, ast.Attribute): + return func.attr + return None + + +def _subprocess_names(tree: ast.AST) -> set[str]: + """Names subprocess is reachable under here, e.g. `import subprocess as _sp`.""" + names = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for alias in node.names: + if alias.name == "subprocess": + names.add(alias.asname or alias.name) + return names + + +def _subprocess_aliases(tree: ast.AST, names: set[str]) -> set[str]: + """Plain names bound to a subprocess callable, called without the module. + + ``install_wheel(run = subprocess.run)`` calls its injected ``run`` as a bare + name, so matching only the attribute form leaves those installer calls + unguarded. Imports, assignments and parameter defaults all bind one. + """ + + def _is_bound(value: ast.expr | None) -> bool: + return ( + isinstance(value, ast.Attribute) + and value.attr in _SUBPROCESS_CALLS + and isinstance(value.value, ast.Name) + and value.value.id in names + ) + + aliases: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom) and node.module == "subprocess": + aliases.update(a.asname or a.name for a in node.names if a.name in _SUBPROCESS_CALLS) + elif isinstance(node, ast.Assign) and _is_bound(node.value): + aliases.update(t.id for t in node.targets if isinstance(t, ast.Name)) + elif isinstance(node, ast.AnnAssign) and _is_bound(node.value): + if isinstance(node.target, ast.Name): + aliases.add(node.target.id) + elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + args = node.args + positional = args.posonlyargs + args.args + # Defaults cover the tail of the positional parameters; kw_defaults + # is aligned with kwonlyargs already, holding None where absent. + padded = [None] * (len(positional) - len(args.defaults)) + list(args.defaults) + pairs = list(zip(positional, padded)) + list(zip(args.kwonlyargs, args.kw_defaults)) + aliases.update(arg.arg for arg, default in pairs if _is_bound(default)) + return aliases + + +def _is_subprocess_call(node: ast.Call, names: set[str], aliases: set[str]) -> bool: + func = node.func + if isinstance(func, ast.Name): + return func.id in aliases + if not isinstance(func, ast.Attribute) or func.attr not in _SUBPROCESS_CALLS: + return False + value = func.value + return isinstance(value, ast.Name) and value.id in names + + +def _text_mode_subprocess(node: ast.Call) -> bool: + for keyword in node.keywords: + if keyword.arg not in ("text", "universal_newlines"): + continue + if isinstance(keyword.value, ast.Constant) and keyword.value.value is True: + return True + return False + + +def _text_mode_dict(node: ast.Dict) -> bool: + """A ``{"text": True, ...}`` literal with no "encoding" key.""" + keys = [k.value for k in node.keys if isinstance(k, ast.Constant)] + if "encoding" in keys: + return False + for key, value in zip(node.keys, node.values): + if not isinstance(key, ast.Constant) or key.value not in ( + "text", + "universal_newlines", + ): + continue + if isinstance(value, ast.Constant) and value.value is True: + return True + return False + + +def _splatted_names(tree: ast.AST) -> set[str]: + """Names handed to a call as ``**name``.""" + names = set() + for node in ast.walk(tree): + if isinstance(node, ast.Call): + for keyword in node.keywords: + if keyword.arg is None and isinstance(keyword.value, ast.Name): + names.add(keyword.value.id) + return names + + +def _encoding_assigned_later(tree: ast.AST, name: str) -> bool: + """``name["encoding"] = ...`` somewhere, so the literal need not carry it.""" + for node in ast.walk(tree): + if not isinstance(node, ast.Subscript) or not isinstance(node.ctx, ast.Store): + continue + target, key = node.value, node.slice + if isinstance(target, ast.Name) and target.id == name: + if isinstance(key, ast.Constant) and key.value == "encoding": + return True + return False + + +def _splatted_kwargs_offenders(tree: ast.AST) -> list[ast.Dict]: + """Text-mode kwargs built in a dict and splatted into a call. + + Kwargs are collected in a dict and splatted (``run(cmd, **run_kwargs)``) + where a branch has to add a timeout or an env, and the call is often through + a helper, so neither the callee nor the keywords are visible at the call + site. Only dicts that reach a call this way are judged: an unrelated payload + that happens to carry ``"text": True`` is not subprocess configuration. + """ + found = [] + # ``run(cmd, **{...})``: the literal is at the call already. + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + for keyword in node.keywords: + if keyword.arg is None and isinstance(keyword.value, ast.Dict): + if _text_mode_dict(keyword.value): + found.append(keyword.value) + splatted = _splatted_names(tree) + if not splatted: + return found + for node in ast.walk(tree): + targets = [] + if isinstance(node, ast.Assign): + targets = [t for t in node.targets if isinstance(t, ast.Name)] + elif isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name): + targets = [node.target] + if not targets or not isinstance(node.value, ast.Dict): + continue + if not _text_mode_dict(node.value): + continue + for target in targets: + if target.id in splatted and not _encoding_assigned_later(tree, target.id): + found.append(node.value) + break + return found + + +def _offenders(path: Path) -> list[str]: + source = path.read_text(encoding = "utf-8") + tree = ast.parse(source, filename = str(path)) + subprocess_names = _subprocess_names(tree) + subprocess_aliases = _subprocess_aliases(tree, subprocess_names) + found: list[str] = [] + for node in _splatted_kwargs_offenders(tree): + found.append( + f"{path.name}:{node.lineno}: subprocess kwargs with text = True and no encoding" + ) + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + name = _call_name(node) + + if _is_subprocess_call(node, subprocess_names, subprocess_aliases): + if _text_mode_subprocess(node) and not _has_keyword(node, "encoding"): + found.append(f"{path.name}:{node.lineno}: subprocess(text = True) without encoding") + continue + + if name == "open" and isinstance(node.func, ast.Name): + if _mode_is_binary(node) or _open_has_encoding(node): + continue + found.append(f"{path.name}:{node.lineno}: open() without encoding") + continue + + # os.fdopen(fd, "w") is open() on a descriptor, so text mode takes the + # same locale default. Its mode defaults to "r", i.e. text, like open's. + if name == "fdopen": + if _mode_is_binary(node) or _open_has_encoding(node): + continue + found.append(f"{path.name}:{node.lineno}: os.fdopen() without encoding") + continue + + if name == "open" and isinstance(node.func, ast.Attribute): + if not _is_path_open(node) or _path_open_has_encoding(node): + continue + if _path_open_mode(node) and "b" in _path_open_mode(node): + continue + found.append(f"{path.name}:{node.lineno}: Path.open() without encoding") + continue + + if name in ("read_text", "write_text") and isinstance(node.func, ast.Attribute): + if _has_keyword(node, "encoding"): + continue + # importlib.metadata Distribution.read_text() takes no encoding kwarg. + if isinstance(node.func.value, ast.Name) and node.func.value.id == "dist": + continue + found.append(f"{path.name}:{node.lineno}: {name}() without encoding") + return found + + +@pytest.mark.parametrize("path", _studio_sources(), ids = lambda p: str(p.name)) +def test_text_io_names_its_encoding(path: Path) -> None: + offenders = _offenders(path) + assert not offenders, ( + "Text I/O without an explicit encoding falls back to the Windows ANSI " + 'codepage and corrupts non-ASCII (ä ö ü → 世). Pass encoding = "utf-8":\n ' + + "\n ".join(offenders) + ) + + +_STATE_STORE = ( + BACKEND_ROOT + / "plugins/data-designer-github-repo-seed/src" + / "data_designer_github_repo_seed/scraper_impl/state_store.py" +) + + +def _load_state_store(codepage: str): + """Load state_store with the writing machine's codepage pinned.""" + spec = importlib.util.spec_from_file_location(f"state_store_{codepage}", _STATE_STORE) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + module.locale = SimpleNamespace( + getencoding = lambda: codepage, + getpreferredencoding = lambda _ = True: codepage, + ) + return module + + +@pytest.mark.parametrize( + ("codepage", "name"), [("cp1252", "Jürgen"), ("cp1251", "Юрий"), ("cp932", "田中")] +) +def test_resuming_a_legacy_jsonl_keeps_one_encoding( + tmp_path: Path, codepage: str, name: str +) -> None: + """A scrape written before UTF-8 was explicit must resume, not duplicate.""" + path = tmp_path / "out.jsonl" + records = [{"id": 1, "author": name}, {"id": 2, "author": name}] + body = "".join(json.dumps(r, ensure_ascii = False) + "\n" for r in records) + path.write_bytes(body.encode(codepage)) + before = path.read_bytes() + + writer = _load_state_store(codepage).JsonlWriter(path) + try: + # Seen keys survive the resume, so a repeat is refused, not appended. + assert writer.has("id:1") and writer.has("id:2") + assert writer.write(records[0]) is False + assert writer.write({"id": 3, "author": name}) is True + finally: + writer.close() + + # Never converted, so it still reads in its own codepage; the append is ASCII. + blob = path.read_bytes() + assert blob.startswith(before) + assert blob[len(before) :].isascii() + lines = [json.loads(x) for x in blob.decode(codepage).splitlines() if x.strip()] + assert len(lines) == 3 + assert [line["author"] for line in lines] == [name] * 3 + + +def test_a_coincidentally_utf8_legacy_line_is_left_alone(tmp_path: Path) -> None: + """cp1251 `Р°` is D0 B0, which is also UTF-8 `а`, and nothing can tell them apart.""" + path = tmp_path / "out.jsonl" + ambiguous = "Р°" + assert ambiguous.encode("cp1251").decode("utf-8") == "а" # the trap + authors = ["Привет", "Здравствуйте", "Москва", ambiguous] + path.write_bytes( + b"".join( + json.dumps({"id": i, "author": a}, ensure_ascii = False).encode("cp1251") + b"\n" + for i, a in enumerate(authors) + ) + ) + before = path.read_bytes() + + _load_state_store("cp1251").JsonlWriter(path).close() + + # Untouched, so the ambiguity never had to be resolved. + assert path.read_bytes() == before + rows = [json.loads(x) for x in path.read_text(encoding = "cp1251").splitlines() if x.strip()] + assert [row["author"] for row in rows] == authors + + +@pytest.mark.parametrize( + ("codepage", "word"), [("cp1251", "Привет"), ("cp932", "こんにちは"), ("cp1252", "Jürgen")] +) +def test_a_moved_shard_is_not_rewritten_by_guesswork( + tmp_path: Path, codepage: str, word: str +) -> None: + """Off the writing machine there is no codepage to attribute the file to.""" + path = tmp_path / "out.jsonl" + # Two records: a lone non-UTF-8 line would count as damage, not legacy. + path.write_bytes( + b"".join( + json.dumps({"id": i, "author": word}, ensure_ascii = False).encode(codepage) + b"\n" + for i in (1, 4) + ) + ) + before = path.read_bytes() + + # A UTF-8 host: latin-1 would read cp1251 `Привет` back as `Ïðèâåò`. + writer = _load_state_store("utf-8").JsonlWriter(path) + try: + assert writer.has("id:1") # ASCII keys still recover + assert writer.write({"id": 2, "author": "Grüße"}) is True + finally: + writer.close() + + blob = path.read_bytes() + assert blob.startswith(before) # never rewritten + assert blob[len(before) :].isascii() # appended as \uXXXX, so no second encoding + rows = [json.loads(x) for x in blob.decode(codepage).splitlines() if x.strip()] + assert [row["author"] for row in rows] == [word, word, "Grüße"] + + +def test_an_all_ambiguous_shard_still_gets_ascii_appends(tmp_path: Path) -> None: + """Every line valid under both readings still means the append must not pick one.""" + path = tmp_path / "out.jsonl" + ambiguous = "Р°" # cp1251 D0 B0, also valid UTF-8 for "а" + path.write_bytes( + b"".join( + json.dumps({"id": i, "a": ambiguous}, ensure_ascii = False).encode("cp1251") + b"\n" + for i in range(3) + ) + ) + before = path.read_bytes() + + writer = _load_state_store("cp1251").JsonlWriter(path) + try: + assert writer.write({"id": 9, "a": "世界"}) is True + finally: + writer.close() + + blob = path.read_bytes() + assert blob.startswith(before) + # ASCII, so the appended record survives whichever reading is chosen. + assert blob[len(before) :].isascii() + for codec in ("cp1251", "utf-8"): + rows = [json.loads(x) for x in blob.decode(codec).splitlines() if x.strip()] + assert rows[-1]["a"] == "世界" + + +def test_a_damaged_line_in_an_ascii_shard_does_not_block_its_retry(tmp_path: Path) -> None: + """With no non-ASCII records to outvote it, one damaged line is still damage.""" + path = tmp_path / "out.jsonl" + path.write_bytes( + b'{"id": 1, "author": "alice"}\n' + + b'{"id": 99, "author": "bad \x96 byte"}\n' + + b'{"id": 2, "author": "bob"}\n' + ) + + writer = _load_state_store("cp1252").JsonlWriter(path) + try: + assert writer.has("id:1") and writer.has("id:2") + assert not writer.has("id:99") + assert writer.write({"id": 99, "author": "good byte"}) is True + finally: + writer.close() + + +def test_a_damaged_line_does_not_block_its_own_retry(tmp_path: Path) -> None: + """Its key comes from the codepage reading, which a UTF-8 shard did not pick.""" + path = tmp_path / "out.jsonl" + path.write_bytes( + json.dumps({"id": 1, "author": "Jürgen"}, ensure_ascii = False).encode() + + b"\n" + + b'{"id": 99, "author": "bad \x96 byte"}\n' + ) + + writer = _load_state_store("cp1252").JsonlWriter(path) + try: + assert writer.has("id:1") + assert not writer.has("id:99") + assert writer.write({"id": 99, "author": "good byte"}) is True + finally: + writer.close() + + +def test_one_damaged_byte_does_not_relabel_a_utf8_shard(tmp_path: Path) -> None: + """A complete JSON line with a stray 0x96 parses as cp1252, but is only one vote.""" + path = tmp_path / "out.jsonl" + healthy = ["Jürgen", "Grüße", "Björn"] + path.write_bytes( + json.dumps({"id": 0, "author": healthy[0]}, ensure_ascii = False).encode() + + b"\n" + + b'{"id": 99, "author": "bad \x96 byte"}\n' + + b"".join( + json.dumps({"id": i, "author": a}, ensure_ascii = False).encode() + b"\n" + for i, a in enumerate(healthy[1:], start = 1) + ) + ) + before = path.read_bytes() + + _load_state_store("cp1252").JsonlWriter(path).close() + + # Untouched, so the healthy records were never re-read as cp1252. + assert path.read_bytes() == before + rows = [] + for line in path.read_bytes().splitlines(): + try: + rows.append(json.loads(line.decode())) + except (UnicodeDecodeError, ValueError): + continue + assert [row["author"] for row in rows] == healthy + + +def test_a_torn_line_does_not_relabel_a_utf8_shard(tmp_path: Path) -> None: + """One interrupted append must not get the whole shard read as cp1252.""" + path = tmp_path / "out.jsonl" + good = [{"id": 1, "author": "Jürgen"}, {"id": 3, "author": "Grüße"}] + torn = '{"id": 2, "author": "Jürgen"}'.encode()[:-6] # cut mid-character + path.write_bytes( + json.dumps(good[0], ensure_ascii = False).encode() + + b"\n" + + torn + + b"\n" + + json.dumps(good[1], ensure_ascii = False).encode() + + b"\n" + ) + before = path.read_bytes() + + writer = _load_state_store("cp1252").JsonlWriter(path) + try: + assert writer.has("id:1") and writer.has("id:3") + assert not writer.has("id:2") # torn line yields no key + finally: + writer.close() + + # Untouched: no rewrite, so no record was re-encoded into mojibake. + after = path.read_bytes() + assert after.startswith(before) + assert "Jürgen".encode() in after + assert "Jürgen".encode("utf-8").decode("cp1252").encode() not in after + + +def test_an_undecodable_transport_marker_reads_as_unknown(tmp_path: Path) -> None: + """Pinning the decode turns an undecodable marker into UnicodeDecodeError, + which is a ValueError and so is not an OSError. Before the pin those bytes + simply read as an unknown value and the caller safely purged and restarted + the partial download; letting the error escape aborts the transfer instead. + """ + import sys + + backend = str(Path(__file__).resolve().parent.parent) + if backend not in sys.path: + sys.path.insert(0, backend) + from hub.utils import download_registry as registry + + marker = tmp_path / ".transport" + marker.write_bytes(b"\x80\xffnative\n") + assert registry._read_marker_value(marker) is None + # A readable but unknown value takes the same path (the behaviour restored). + marker.write_text("something-else\n", encoding = "utf-8") + assert registry._read_marker_value(marker) is None + + +def test_a_torn_cache_ref_reads_as_not_cached(tmp_path: Path, monkeypatch) -> None: + """hf_cache_snapshot_dir answers "is this model already on disk", and the + offline embedding checks turn a raise into a 500. A refs/main holding a byte + the codepage used to decode into a nonsense commit simply missed the snapshot + dir before the pin; it has to keep missing it.""" + import sys + + backend = str(Path(__file__).resolve().parent.parent) + if backend not in sys.path: + sys.path.insert(0, backend) + from utils import utils as backend_utils + + good_root = tmp_path / "good" + torn_root = tmp_path / "torn" + for root, ref_bytes in ((torn_root, b"\x80\xff\n"), (good_root, b"abc123\n")): + repo = root / "models--Org--Model" + (repo / "refs").mkdir(parents = True) + (repo / "refs" / "main").write_bytes(ref_bytes) + (good_root / "models--Org--Model" / "snapshots" / "abc123").mkdir(parents = True) + + monkeypatch.setattr(backend_utils, "_hf_cache_roots", lambda: [torn_root]) + assert backend_utils.hf_cache_snapshot_dir("Org/Model") is None + # The torn root is skipped, not fatal: a healthy second root still answers. + monkeypatch.setattr(backend_utils, "_hf_cache_roots", lambda: [torn_root, good_root]) + found = backend_utils.hf_cache_snapshot_dir("Org/Model") + assert found is not None and found.name == "abc123" + + +def test_a_corrupt_pid_file_does_not_abort_shutdown(tmp_path: Path, monkeypatch) -> None: + """_remove_pid_file runs first in _graceful_shutdown, so a raise there leaves + the inference, export, training and tunnel children alive.""" + import sys + + backend = str(Path(__file__).resolve().parent.parent) + if backend not in sys.path: + sys.path.insert(0, backend) + import run as studio_run + + pid_file = tmp_path / "studio.pid" + pid_file.write_bytes(b"\x80\xff") + monkeypatch.setattr(studio_run, "_PID_FILE", pid_file) + studio_run._remove_pid_file() + # Not this process's PID, so the file stays; the point is that it returned. + assert pid_file.exists() + + pid_file.write_text(str(os.getpid()), encoding = "utf-8") + studio_run._remove_pid_file() + assert not pid_file.exists() + + +def test_the_kwargs_guard_only_judges_dicts_that_reach_a_call(tmp_path: Path) -> None: + """Only a dict splatted into a call is subprocess configuration. An unrelated + payload that happens to carry "text": True is not, and neither is one whose + encoding is filled in on a later line.""" + cases = { + "offender.py": 'kw = {"text": True}\nrun(cmd, **kw)\n', + "annotated.py": 'kw: dict = {"universal_newlines": True}\nrun(cmd, **kw)\n', + "payload.py": 'payload = {"text": True}\nrequests.post(url, json = payload)\n', + "inline.py": 'run(cmd, **{"text": True})\n', + "later.py": 'kw = {"text": True}\nkw["encoding"] = "utf-8"\nrun(cmd, **kw)\n', + "carried.py": 'kw = {"text": True, "encoding": "utf-8"}\nrun(cmd, **kw)\n', + } + flagged = set() + for name, source in cases.items(): + path = tmp_path / name + path.write_text(source, encoding = "utf-8") + if any("subprocess kwargs" in line for line in _offenders(path)): + flagged.add(name) + assert flagged == {"offender.py", "annotated.py", "inline.py"}, flagged + + +def test_the_guard_follows_subprocess_through_an_alias(tmp_path: Path) -> None: + """install_wheel() takes ``run = subprocess.run`` and calls it as a bare + name, so an attribute-only match let both of its installer calls drop their + encoding unnoticed. A name bound to something else is still not subprocess.""" + cases = { + "param_default.py": ( + "import subprocess\n" + "def install(*, run = subprocess.run):\n" + " run(cmd, text = True)\n" + ), + "assigned.py": "import subprocess\n_run = subprocess.run\n_run(cmd, text = True)\n", + "imported.py": "from subprocess import check_output\ncheck_output(cmd, text = True)\n", + "renamed.py": "from subprocess import run as _r\n_r(cmd, universal_newlines = True)\n", + "encoded.py": ( + "import subprocess\n" + "def install(*, run = subprocess.run):\n" + ' run(cmd, text = True, encoding = "utf-8")\n' + ), + "unrelated.py": "def run(cmd, text = False):\n pass\nrun(cmd, text = True)\n", + } + flagged = set() + for name, source in cases.items(): + path = tmp_path / name + path.write_text(source, encoding = "utf-8") + if any("subprocess(text = True)" in line for line in _offenders(path)): + flagged.add(name) + assert flagged == {"param_default.py", "assigned.py", "imported.py", "renamed.py"}, flagged + + +def test_the_guard_sees_os_fdopen(tmp_path: Path) -> None: + """os.fdopen(fd, mode) is open() on a descriptor and takes the same locale + default in text mode, so leaving it out let the swap lock file keep the + codepage on the write side while its reader was pinned to UTF-8.""" + cases = { + "text.py": 'import os\nos.fdopen(fd, "w")\n', + "default_mode.py": "import os\nos.fdopen(fd)\n", # defaults to "r", still text + "binary.py": 'import os\nos.fdopen(fd, "wb")\n', + "keyword.py": 'import os\nos.fdopen(fd, "w", encoding = "utf-8")\n', + "positional.py": 'import os\nos.fdopen(fd, "w", 1, "utf-8")\n', + } + flagged = set() + for name, source in cases.items(): + path = tmp_path / name + path.write_text(source, encoding = "utf-8") + if any("fdopen" in line for line in _offenders(path)): + flagged.add(name) + assert flagged == {"text.py", "default_mode.py"}, flagged + + +def test_an_undecodable_bootstrap_password_does_not_stop_startup( + tmp_path: Path, monkeypatch +) -> None: + """ensure_default_admin calls _load_bootstrap_password for every existing + admin and the lifespan calls that with no handler, so a raise here takes the + whole backend down instead of ignoring an unusable file.""" + import sys + + backend = str(Path(__file__).resolve().parent.parent) + if backend not in sys.path: + sys.path.insert(0, backend) + from auth import storage + + pw_file = tmp_path / ".bootstrap_password" + pw_file.write_bytes(b"\x80\xffnot-utf8\n") + monkeypatch.setattr(storage, "_BOOTSTRAP_PW_PATH", pw_file) + assert storage._load_bootstrap_password() is None + + # A readable one still loads, so this is a narrowing of failure, not of function. + pw_file.write_text("correct horse battery staple\n", encoding = "utf-8") + assert storage._load_bootstrap_password() == "correct horse battery staple" + + +def test_a_damaged_checkpoint_resets_instead_of_resuming_on_a_broken_cursor(tmp_path: Path) -> None: + """A checkpoint holds only base64 cursors and booleans, so a codepage reading + can only ever add non-ASCII, never recover any. Resuming on a mojibaked cursor + sends GitHub one it answers with INVALID_CURSOR_ARGUMENTS, and the empty page + that comes back marks the stream done and skips the rest of it for good. + Dropping the checkpoint only replays pages the writers already dedup.""" + module = _load_state_store("cp1252") + cursor = "Y3Vyc29yOnYyOpK0MjAxMi0wMi0xNlQwNjo1Mzo0MVrOADGL_A==" + healthy = json.dumps({"issues_cursor": cursor, "issues_done": False}, indent = 2) + path = tmp_path / "octocat__Hello-World.json" + + path.write_text(healthy, encoding = "utf-8") + assert module.StateStore(path).get("issues_cursor") == cursor + + # Written by a pre-UTF-8 release in the operator's codepage. Nothing is lost + # by reading UTF-8 only, because an all-ASCII document is the same bytes. + path.write_bytes(healthy.encode("cp1252")) + assert module.StateStore(path).get("issues_cursor") == cursor + + # One damaged byte inside the cursor: still a whole JSON document under a + # single-byte codepage, so only refusing that reading resets the checkpoint. + raw = healthy.encode() + at = raw.index(b"MjAxMi0wMi0xNlQ") + 3 + path.write_bytes(raw[:at] + b"\x96" + raw[at + 1 :]) + assert json.loads(path.read_bytes().decode("latin-1"))["issues_cursor"] != cursor + store = module.StateStore(path) + assert store.all() == {} + assert store.get("issues_cursor") is None + + +def test_a_utf8_record_is_not_parsed_a_second_time(tmp_path: Path) -> None: + """These shards reach gigabytes and every resume reads all of one, so a + record that already read as UTF-8 must not be decoded and parsed again under + the codepage. The legacy reading exists only to recover keys UTF-8 could not.""" + module = _load_state_store("cp1252") + calls: list[str] = [] + real_parse = module._parse + + def counting_parse(raw, encoding): + calls.append(encoding) + return real_parse(raw, encoding) + + module._parse = counting_parse + try: + healthy = json.dumps({"id": 1, "author": "Jürgen"}).encode("utf-8") + reading = module._read_line(healthy, "cp1252") + assert reading.as_utf8 == {"id": 1, "author": "Jürgen"} + assert calls == ["utf-8"], calls + + # A line UTF-8 cannot read still falls through to the codepage, the whole point. + calls.clear() + legacy = json.dumps({"id": 2, "author": "Jürgen"}, ensure_ascii = False).encode("cp1252") + reading = module._read_line(legacy, "cp1252") + assert reading.as_utf8 is None + assert reading.as_legacy == {"id": 2, "author": "Jürgen"} + assert calls == ["utf-8", "cp1252"], calls + finally: + module._parse = real_parse + + +def _too_deeply_nested_json() -> str: + """A JSON document nested past what this interpreter will descend into. + + Probed rather than hardcoded: the depth json.loads gives up at is bounded by + sys.getrecursionlimit() up to 3.11 and by the C recursion limit from 3.12, + which sys.setrecursionlimit no longer moves and which varies by micro + version. That is ~995 on 3.9 and ~9999 on 3.13. + """ + depth = 1 + while depth <= 1 << 17: + document = "[" * depth + "]" * depth + try: + json.loads(document) + except RecursionError: + return document + depth *= 2 + pytest.skip("this interpreter parses arbitrarily nested JSON") + + +def test_an_unparseably_nested_document_is_discarded_not_raised(tmp_path: Path) -> None: + """json.loads answers nesting it cannot descend with RecursionError, which is + a RuntimeError and so is neither a ValueError nor a UnicodeDecodeError. + _parse is called outside any other handler in both StateStore.__init__ and + JsonlWriter._scan_existing, so letting it escape aborts the scraper at + startup on a file the catch-all it replaced simply discarded.""" + module = _load_state_store("cp1252") + nested = _too_deeply_nested_json() + + checkpoint = tmp_path / "octocat__Hello-World.json" + checkpoint.write_text(nested, encoding = "utf-8") + assert module.StateStore(checkpoint).all() == {} # reset, not raised + + shard = tmp_path / "out.jsonl" + shard.write_text( + nested + "\n" + json.dumps({"id": 1}) + "\n" + json.dumps({"id": 2}) + "\n", + encoding = "utf-8", + ) + writer = module.JsonlWriter(shard) + try: + # Skipped like any other unreadable line, so its neighbours still yield the dedup + # keys that keep the resume from re-fetching them. + assert writer.has("id:1") and writer.has("id:2") + finally: + writer.close() diff --git a/studio/backend/tests/test_training_worker_flash_attn.py b/studio/backend/tests/test_training_worker_flash_attn.py index 86511987b1..d136821ea2 100644 --- a/studio/backend/tests/test_training_worker_flash_attn.py +++ b/studio/backend/tests/test_training_worker_flash_attn.py @@ -9,8 +9,28 @@ import sys from typing import Any from unittest import mock +import pytest + from core.training import worker +# The runtime install is Linux-only, so elsewhere these return before any status. +linux_only = pytest.mark.skipif( + not sys.platform.startswith("linux"), + reason = "the runtime flash-attn install is gated to Linux", +) + +# causal-conv1d and flash-linear-attention are NOT Linux-gated: both installers bail out +# on `sys.platform == "win32"` alone (no prebuilt wheel for Windows) and run everywhere +# else, macOS included. linux_only here would skip cases that legitimately pass off Linux. +not_on_windows = pytest.mark.skipif( + sys.platform == "win32", + reason = ( + "mirrors the sys.platform == 'win32' bail-out in " + "_ensure_flash_linear_attention_unconditional and " + "_ensure_causal_conv1d_fast_path" + ), +) + def _missing_flash_attn_import(): real_import = builtins.__import__ @@ -55,6 +75,7 @@ def test_should_try_runtime_flash_attn_install_threshold_and_skip(monkeypatch): assert worker._should_try_runtime_flash_attn_install(32768) is False +@linux_only def test_runtime_flash_attn_prefers_prebuilt_wheel(monkeypatch): statuses: list[str] = [] @@ -82,6 +103,7 @@ def test_runtime_flash_attn_prefers_prebuilt_wheel(monkeypatch): assert statuses == ["Installing flash-attn for faster training..."] +@linux_only def test_runtime_flash_attn_falls_back_to_pypi(monkeypatch): calls: list[list[str]] = [] statuses: list[str] = [] @@ -113,12 +135,7 @@ def test_runtime_flash_attn_falls_back_to_pypi(monkeypatch): ) monkeypatch.setattr(worker, "install_wheel", mock.Mock()) - def fake_run( - cmd, - stdout = None, - stderr = None, - text = None, - ): + def fake_run(cmd, **kwargs): calls.append(list(cmd)) return subprocess.CompletedProcess(cmd, 0, "") @@ -139,6 +156,7 @@ def test_runtime_flash_attn_skip_env_avoids_all_install_work(monkeypatch): worker._sp.run.assert_not_called() +@not_on_windows def test_causal_conv1d_fast_path_preserves_wheel_first_install_args(monkeypatch): install_mock = mock.Mock(return_value = True) monkeypatch.setattr(worker, "_install_package_wheel_first", install_mock) @@ -160,6 +178,7 @@ def test_causal_conv1d_fast_path_preserves_wheel_first_install_args(monkeypatch) ) +@not_on_windows def test_causal_conv1d_fast_path_includes_qwen3_6_variants(monkeypatch): install_mock = mock.Mock(return_value = True) monkeypatch.setattr(worker, "_install_package_wheel_first", install_mock) @@ -225,6 +244,7 @@ def _pin_fla_model_types(monkeypatch): ) +@not_on_windows def test_flash_linear_attention_installs_pinned_pair_for_qwen3_5(monkeypatch): _pin_fla_model_types(monkeypatch) monkeypatch.setattr(worker.shutil, "which", lambda name: "/usr/bin/uv") @@ -277,6 +297,7 @@ def test_flash_linear_attention_skips_for_ssm_only_models(monkeypatch): run_mock.assert_not_called() +@not_on_windows def test_flash_linear_attention_matches_full_qwen3_family(monkeypatch): monkeypatch.setattr(worker.shutil, "which", lambda name: "/usr/bin/uv") run_mock = mock.Mock(return_value = mock.Mock(returncode = 0, stdout = "")) @@ -331,6 +352,7 @@ def test_flash_linear_attention_skipped_via_env(monkeypatch): run_mock.assert_not_called() +@not_on_windows def test_flash_linear_attention_skipped_below_torch_2_7(monkeypatch): _pin_fla_model_types(monkeypatch) monkeypatch.delenv(worker._FLA_SKIP_ENV, raising = False) @@ -349,6 +371,7 @@ def test_flash_linear_attention_skipped_below_torch_2_7(monkeypatch): assert any("torch>=" in s for s in statuses) +@not_on_windows def test_flash_linear_attention_install_includes_einops(monkeypatch): _pin_fla_model_types(monkeypatch) monkeypatch.delenv(worker._FLA_SKIP_ENV, raising = False) @@ -375,6 +398,7 @@ def test_flash_linear_attention_install_includes_einops(monkeypatch): assert f"fla-core=={worker._FLA_CORE_PACKAGE_VERSION}" in args +@not_on_windows def test_flash_linear_attention_logs_post_install_import_failure(monkeypatch): """pip exits 0 but `import fla.modules` still fails (missing transitive).""" _pin_fla_model_types(monkeypatch) @@ -421,6 +445,7 @@ def test_tilelang_backend_skipped_on_unsupported_linux_arch(monkeypatch): run_mock.assert_not_called() +@linux_only def test_tilelang_backend_pins_only_binary(monkeypatch): _pin_fla_model_types(monkeypatch) monkeypatch.delenv(worker._TILELANG_SKIP_ENV, raising = False) @@ -462,6 +487,7 @@ def _force_missing_tilelang_imports(monkeypatch): monkeypatch.setattr(builtins, "__import__", fake_import) +@linux_only def test_tilelang_backend_installs_pinned_pair_for_qwen3_5(monkeypatch): _pin_fla_model_types(monkeypatch) monkeypatch.delenv(worker._TILELANG_SKIP_ENV, raising = False) @@ -486,6 +512,7 @@ def test_tilelang_backend_installs_pinned_pair_for_qwen3_5(monkeypatch): assert any("Installing TileLang" in s for s in statuses) +@linux_only def test_tilelang_backend_reinstalls_when_tvm_ffi_is_broken(monkeypatch): """Repair path issues TWO pip calls: @@ -555,6 +582,7 @@ def test_tilelang_backend_skipped_on_windows(monkeypatch): run_mock.assert_not_called() +@linux_only def test_tilelang_backend_swallows_install_timeout(monkeypatch): _pin_fla_model_types(monkeypatch) monkeypatch.delenv(worker._TILELANG_SKIP_ENV, raising = False) @@ -609,6 +637,7 @@ def test_tilelang_backend_skipped_via_env(monkeypatch): run_mock.assert_not_called() +@linux_only def test_tilelang_backend_swallows_install_failure(monkeypatch): _pin_fla_model_types(monkeypatch) monkeypatch.delenv(worker._TILELANG_SKIP_ENV, raising = False) @@ -673,6 +702,7 @@ def _patch_iu_gates(monkeypatch, fla_gate, conv_gate): monkeypatch.setattr(_iu, "is_causal_conv1d_available", conv_gate) +@not_on_windows def test_hook_installs_when_gate_returns_false(monkeypatch): _pin_fla_model_types(monkeypatch) fla_gate = _make_fake_gate(initial_return = False) @@ -976,6 +1006,7 @@ def test_hook_does_install_tilelang_for_qwen35(monkeypatch): tile_install.assert_called_once() +@linux_only def test_tilelang_repair_does_not_touch_torch_cuda_stack(monkeypatch): """Finding #2: the broken-tvm-ffi repair must use --no-deps on the forced step so --force-reinstall doesn't cascade through @@ -1119,6 +1150,7 @@ def test_hook_runs_tilelang_repair_when_fla_already_true(monkeypatch): tile_install.assert_called_once() +@not_on_windows def test_fla_installer_force_reinstalls_when_older_version_present(monkeypatch): """Finding #8: an older `flash-linear-attention` that is importable but below the pin must force a reinstall (not no-op). @@ -1583,15 +1615,10 @@ def test_install_respects_user_gcc_install_dir(monkeypatch): ) _make_hip_install_env(monkeypatch, gcc_dir = "/usr/lib/gcc/x86_64-linux-gnu/13") - captured: dict[str, str] | None = {"_called": "no"} + captured: dict[str, str] = {} def fake_run(cmd, **kwargs): - env = kwargs.get("env") - if env is not None: - captured.clear() - captured.update(env) - else: - captured["_called"] = "yes_no_env" + captured.update(kwargs.get("env") or {}) return subprocess.CompletedProcess(cmd, 0, "") monkeypatch.setattr(worker._sp, "run", fake_run) @@ -1607,14 +1634,11 @@ def test_install_respects_user_gcc_install_dir(monkeypatch): release_base_url = "https://example.com", ) - # subprocess.run invoked without env override (user already set - # HIPCC_COMPILE_FLAGS_APPEND with --gcc-install-dir, so we left the - # env alone — the existing value is inherited). - assert captured == {"_called": "yes_no_env"} + assert captured["HIPCC_COMPILE_FLAGS_APPEND"] == "--gcc-install-dir=/opt/custom/gcc-13" def test_install_does_not_inject_env_on_cuda(monkeypatch): - """CUDA path (no hip_version in env) → no env override at all.""" + """CUDA path (no hip_version in env) → no HIP flag injected.""" monkeypatch.delenv("HIPCC_COMPILE_FLAGS_APPEND", raising = False) monkeypatch.setattr(builtins, "__import__", _missing_module_import("causal_conv1d")) monkeypatch.setattr( @@ -1641,7 +1665,7 @@ def test_install_does_not_inject_env_on_cuda(monkeypatch): captured: dict[str, Any] = {} def fake_run(cmd, **kwargs): - captured["env_in_kwargs"] = "env" in kwargs + captured.update(kwargs.get("env") or {}) return subprocess.CompletedProcess(cmd, 0, "") monkeypatch.setattr(worker._sp, "run", fake_run) @@ -1657,5 +1681,5 @@ def test_install_does_not_inject_env_on_cuda(monkeypatch): release_base_url = "https://example.com", ) - # CUDA branch never sets the env, never invokes the gcc helper. - assert captured.get("env_in_kwargs") is False + # env is always passed (to force UTF-8), but never the HIP flag. + assert "HIPCC_COMPILE_FLAGS_APPEND" not in captured diff --git a/studio/backend/utils/child_stdio.py b/studio/backend/utils/child_stdio.py new file mode 100644 index 0000000000..4709d650df --- /dev/null +++ b/studio/backend/utils/child_stdio.py @@ -0,0 +1,22 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Make a Python child agree with the parent that its pipes are UTF-8. + +A child's ``sys.stdout`` uses ``locale.getpreferredencoding()``, which on +Windows is the ANSI code page. Reading that pipe as UTF-8 would then mangle any +non-ASCII the child prints, so the child has to be told which encoding to emit. +Only needed for Python children; llama.cpp and node already emit UTF-8. +""" + +from __future__ import annotations + +import os +from typing import Mapping, Optional + + +def utf8_child_env(env: Optional[Mapping[str, str]] = None) -> dict[str, str]: + """Copy *env* (or the current environment) with UTF-8 stdio forced.""" + child = dict(os.environ if env is None else env) + child["PYTHONIOENCODING"] = "utf-8" + return child diff --git a/studio/backend/utils/hardware/amd.py b/studio/backend/utils/hardware/amd.py index 91a06c9a2a..318759f67d 100644 --- a/studio/backend/utils/hardware/amd.py +++ b/studio/backend/utils/hardware/amd.py @@ -144,6 +144,8 @@ def _run_amd_smi(*args: str, timeout: int = _AMD_SMI_DEFAULT_TIMEOUT) -> Optiona ["amd-smi", *args, "--json"], capture_output = True, text = True, + encoding = "utf-8", + errors = "replace", timeout = timeout, env = _amd_env, **windows_hidden_subprocess_kwargs(), diff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.py index 48ba375ec5..300d26c362 100644 --- a/studio/backend/utils/hardware/hardware.py +++ b/studio/backend/utils/hardware/hardware.py @@ -830,6 +830,8 @@ def _rocm_windows_perf_counter_gpu_util_pct() -> Optional[float]: ["powershell", "-NoProfile", "-NonInteractive", "-Command", ps], capture_output = True, text = True, + encoding = "utf-8", + errors = "replace", timeout = 5, ) if r.returncode != 0 or not r.stdout.strip(): @@ -1027,6 +1029,8 @@ def _rocm_windows_perf_counter_vram_by_adapter() -> Optional[list[tuple[str, flo ["powershell", "-NoProfile", "-NonInteractive", "-Command", ps], capture_output = True, text = True, + encoding = "utf-8", + errors = "replace", timeout = 5, ) if r.returncode != 0 or not r.stdout.strip(): diff --git a/studio/backend/utils/hardware/nvidia.py b/studio/backend/utils/hardware/nvidia.py index f98ca4343e..39e3652921 100644 --- a/studio/backend/utils/hardware/nvidia.py +++ b/studio/backend/utils/hardware/nvidia.py @@ -55,6 +55,8 @@ def get_physical_gpu_count() -> Optional[int]: ["nvidia-smi", "-L"], capture_output = True, text = True, + encoding = "utf-8", + errors = "replace", timeout = 5, env = child_env_without_native_path_secret(), **_windows_hidden_subprocess_kwargs(), @@ -81,6 +83,8 @@ def get_primary_gpu_utilization() -> dict[str, Any]: ], capture_output = True, text = True, + encoding = "utf-8", + errors = "replace", timeout = 5, env = child_env_without_native_path_secret(), **_windows_hidden_subprocess_kwargs(), @@ -131,6 +135,8 @@ def get_visible_gpu_utilization( ], capture_output = True, text = True, + encoding = "utf-8", + errors = "replace", timeout = 5, env = child_env_without_native_path_secret(), **_windows_hidden_subprocess_kwargs(), @@ -215,6 +221,8 @@ def get_backend_visible_gpu_info( ], capture_output = True, text = True, + encoding = "utf-8", + errors = "replace", timeout = 10, env = child_env_without_native_path_secret(), **_windows_hidden_subprocess_kwargs(), diff --git a/studio/backend/utils/llama_cpp_update.py b/studio/backend/utils/llama_cpp_update.py index dffcddb452..5c9646f4eb 100644 --- a/studio/backend/utils/llama_cpp_update.py +++ b/studio/backend/utils/llama_cpp_update.py @@ -121,7 +121,14 @@ def _installed_build_number(binary: Optional[str]) -> Optional[int]: if not binary: return None try: - proc = subprocess.run([binary, "--version"], capture_output = True, text = True, timeout = 20) + proc = subprocess.run( + [binary, "--version"], + capture_output = True, + text = True, + encoding = "utf-8", + errors = "replace", + timeout = 20, + ) except Exception: # pragma: no cover - defensive return None m = re.search(r"version:\s*(\d+)", (proc.stderr or "") + (proc.stdout or "")) diff --git a/studio/backend/utils/mlx_repair.py b/studio/backend/utils/mlx_repair.py index 4ea1ec62f5..8e2a6a7712 100644 --- a/studio/backend/utils/mlx_repair.py +++ b/studio/backend/utils/mlx_repair.py @@ -254,7 +254,7 @@ def _transformers_constraint_args() -> tuple[list[str], str | None]: except Exception: return [], None fd, path = tempfile.mkstemp(prefix = "mlx_repair_", suffix = ".txt") - with os.fdopen(fd, "w") as fh: + with os.fdopen(fd, "w", encoding = "utf-8") as fh: fh.write(f"transformers=={transformers_version}\n") return ["--constraint", path], path @@ -290,6 +290,8 @@ def attempt_mlx_repair(*, timeout: int = _REPAIR_TIMEOUT_S) -> bool: stdout = subprocess.PIPE, stderr = subprocess.STDOUT, text = True, + encoding = "utf-8", + errors = "replace", timeout = timeout, ) except subprocess.TimeoutExpired: diff --git a/studio/backend/utils/models/checkpoints.py b/studio/backend/utils/models/checkpoints.py index 6950667bbd..eaf75140fc 100644 --- a/studio/backend/utils/models/checkpoints.py +++ b/studio/backend/utils/models/checkpoints.py @@ -129,7 +129,7 @@ def _read_checkpoint_loss(checkpoint_path: Path) -> Optional[float]: if not trainer_state.exists(): return None try: - with open(trainer_state, encoding = "utf-8") as f: + with open(trainer_state, encoding = "utf-8-sig") as f: state = json.load(f) log_history = state.get("log_history", []) if log_history: @@ -174,18 +174,18 @@ def scan_checkpoints( metadata: dict = {} try: if adapter_config.exists(): - cfg = json.loads(adapter_config.read_text(encoding = "utf-8")) + cfg = json.loads(adapter_config.read_text(encoding = "utf-8-sig")) metadata["base_model"] = cfg.get("base_model_name_or_path") metadata["peft_type"] = cfg.get("peft_type") metadata["lora_rank"] = cfg.get("r") elif config_file.exists(): - cfg = json.loads(config_file.read_text(encoding = "utf-8")) + cfg = json.loads(config_file.read_text(encoding = "utf-8-sig")) metadata["base_model"] = cfg.get("_name_or_path") # Detect BNB quantization from config.json if config_file.exists(): if "cfg" not in dir(): - cfg = json.loads(config_file.read_text(encoding = "utf-8")) + cfg = json.loads(config_file.read_text(encoding = "utf-8-sig")) quant_cfg = cfg.get("quantization_config") if ( isinstance(quant_cfg, dict) diff --git a/studio/backend/utils/models/model_config.py b/studio/backend/utils/models/model_config.py index 893b842e11..6270d9e03f 100644 --- a/studio/backend/utils/models/model_config.py +++ b/studio/backend/utils/models/model_config.py @@ -37,6 +37,7 @@ import yaml from utils.native_path_leases import child_env_without_native_path_secret +from utils.child_stdio import utf8_child_env from utils.hf_cache_settings import active_hf_hub_cache, get_hf_cache_paths from utils.subprocess_compat import ( windows_hidden_subprocess_kwargs as _windows_hidden_subprocess_kwargs, @@ -631,7 +632,7 @@ def _raw_config_has_vision_config( cache_dir = active_hf_hub_cache(), ) ) - config = json.loads(config_path.read_text(encoding = "utf-8")) + config = json.loads(config_path.read_text(encoding = "utf-8-sig")) architectures = config.get("architectures") or [] model_type = config.get("model_type") explicit_vision = ( @@ -774,8 +775,12 @@ def _is_vision_model_subprocess(model_name: str, hf_token: Optional[str] = None) ], capture_output = True, text = True, + encoding = "utf-8", + errors = "replace", timeout = 60, - env = get_hf_cache_paths().child_env(child_env_without_native_path_secret()), + env = utf8_child_env( + get_hf_cache_paths().child_env(child_env_without_native_path_secret()) + ), **_windows_hidden_subprocess_kwargs(), ) @@ -1083,7 +1088,7 @@ def _detect_audio_from_tokenizer( ]: tok_file = snapshot / tok_path if tok_file.exists(): - tok_config = json.loads(tok_file.read_text(encoding = "utf-8")) + tok_config = json.loads(tok_file.read_text(encoding = "utf-8-sig")) read_any = True result = _check_token_patterns(tok_config) if result: @@ -2283,7 +2288,7 @@ def scan_exported_models( export_meta = run_dir / "export_metadata.json" try: if export_meta.exists(): - meta = json.loads(export_meta.read_text(encoding = "utf-8")) + meta = json.loads(export_meta.read_text(encoding = "utf-8-sig")) base_model = meta.get("base_model") except Exception: pass @@ -2312,7 +2317,7 @@ def scan_exported_models( if adapter_config.exists(): export_type = "lora" try: - cfg = json.loads(adapter_config.read_text(encoding = "utf-8")) + cfg = json.loads(adapter_config.read_text(encoding = "utf-8-sig")) base_model = cfg.get("base_model_name_or_path") except Exception: pass @@ -2321,7 +2326,7 @@ def scan_exported_models( export_meta = checkpoint_dir / "export_metadata.json" try: if export_meta.exists(): - meta = json.loads(export_meta.read_text(encoding = "utf-8")) + meta = json.loads(export_meta.read_text(encoding = "utf-8-sig")) base_model = meta.get("base_model") except Exception: pass @@ -2334,7 +2339,7 @@ def scan_exported_models( export_meta = meta_dir / "export_metadata.json" try: if export_meta.exists(): - meta = json.loads(export_meta.read_text(encoding = "utf-8")) + meta = json.loads(export_meta.read_text(encoding = "utf-8-sig")) base_model = meta.get("base_model") if base_model: break @@ -2354,7 +2359,7 @@ def scan_exported_models( outputs_adapter_cfg = resolve_output_dir(run_dir.name) / "adapter_config.json" try: if outputs_adapter_cfg.exists(): - cfg = json.loads(outputs_adapter_cfg.read_text(encoding = "utf-8")) + cfg = json.loads(outputs_adapter_cfg.read_text(encoding = "utf-8-sig")) base_model = cfg.get("base_model_name_or_path") except Exception: pass @@ -2380,7 +2385,7 @@ def get_base_model_from_checkpoint(checkpoint_path: str) -> Optional[str]: adapter_config_path = checkpoint_path_obj / "adapter_config.json" if adapter_config_path.exists(): - with open(adapter_config_path, "r", encoding = "utf-8") as f: + with open(adapter_config_path, "r", encoding = "utf-8-sig") as f: config = json.load(f) base_model = config.get("base_model_name_or_path") if base_model: @@ -2389,7 +2394,7 @@ def get_base_model_from_checkpoint(checkpoint_path: str) -> Optional[str]: config_path = checkpoint_path_obj / "config.json" if config_path.exists(): - with open(config_path, "r", encoding = "utf-8") as f: + with open(config_path, "r", encoding = "utf-8-sig") as f: config = json.load(f) for key in ("model_name", "_name_or_path"): base_model = config.get(key) @@ -2445,7 +2450,7 @@ def get_base_model_from_lora(lora_path: str) -> Optional[str]: # adapter_config.json first adapter_config_path = lora_path_obj / "adapter_config.json" if adapter_config_path.exists(): - with open(adapter_config_path, "r", encoding = "utf-8") as f: + with open(adapter_config_path, "r", encoding = "utf-8-sig") as f: config = json.load(f) base_model = config.get("base_model_name_or_path") if base_model: @@ -2535,7 +2540,7 @@ def get_base_model_from_lora_identifier( last_exc = exc continue try: - with open(cfg_path, "r", encoding = "utf-8") as f: + with open(cfg_path, "r", encoding = "utf-8-sig") as f: base_model = json.load(f).get("base_model_name_or_path") except Exception as exc: logger.warning("Could not parse adapter_config.json for '%s': %s", identifier, exc) @@ -2781,7 +2786,7 @@ class ModelConfig: meta_path = gguf_dir / "export_metadata.json" if meta_path.exists(): try: - meta = json.loads(meta_path.read_text(encoding = "utf-8")) + meta = json.loads(meta_path.read_text(encoding = "utf-8-sig")) base = meta.get("base_model") if base and is_vision_model(base, hf_token = hf_token): base_is_vision = True @@ -2912,7 +2917,7 @@ class ModelConfig: token = hf_token, cache_dir = active_hf_hub_cache(), ) - with open(config_path, "r", encoding = "utf-8") as f: + with open(config_path, "r", encoding = "utf-8-sig") as f: adapter_config = json.load(f) base_model = adapter_config.get("base_model_name_or_path") if base_model: diff --git a/studio/backend/utils/node_runtime.py b/studio/backend/utils/node_runtime.py index fef2430708..697661a095 100644 --- a/studio/backend/utils/node_runtime.py +++ b/studio/backend/utils/node_runtime.py @@ -79,6 +79,8 @@ def _node_version_ok(executable: str) -> bool: [executable, "-v"], capture_output = True, text = True, + encoding = "utf-8", + errors = "replace", timeout = _NODE_VERSION_PROBE_TIMEOUT_SECONDS, **windows_hidden_subprocess_kwargs(), ) diff --git a/studio/backend/utils/paths/storage_roots.py b/studio/backend/utils/paths/storage_roots.py index ae1319d296..0b1398f6d2 100644 --- a/studio/backend/utils/paths/storage_roots.py +++ b/studio/backend/utils/paths/storage_roots.py @@ -212,7 +212,7 @@ def lmstudio_model_dirs() -> list[Path]: settings_path = Path.home() / ".lmstudio" / "settings.json" if settings_path.is_file(): try: - with open(settings_path, encoding = "utf-8") as f: + with open(settings_path, encoding = "utf-8-sig") as f: settings = json.load(f) downloads = settings.get("downloadsFolder", "") if downloads: diff --git a/studio/backend/utils/prebuilt/update_flow.py b/studio/backend/utils/prebuilt/update_flow.py index 74af0c18f9..69c1566fc3 100644 --- a/studio/backend/utils/prebuilt/update_flow.py +++ b/studio/backend/utils/prebuilt/update_flow.py @@ -24,6 +24,7 @@ from typing import Callable, Optional import structlog +from utils.child_stdio import utf8_child_env from utils.process_lifetime import child_popen_kwargs logger = structlog.get_logger(__name__) @@ -159,6 +160,8 @@ def resolve_prebuilt_for_host( cmd, capture_output = True, text = True, + encoding = "utf-8", + errors = "replace", timeout = 60, ) out = (proc.stdout or "").strip() @@ -303,7 +306,10 @@ def stream_installer( stdout = subprocess.PIPE, stderr = subprocess.STDOUT, text = True, - env = env, + encoding = "utf-8", + errors = "replace", + # Make the Python child emit the UTF-8 we decode above. + env = utf8_child_env(env), **child_popen_kwargs(), ) timed_out = threading.Event() diff --git a/studio/backend/utils/security/consent.py b/studio/backend/utils/security/consent.py index 6fee259139..9385270ee0 100644 --- a/studio/backend/utils/security/consent.py +++ b/studio/backend/utils/security/consent.py @@ -142,7 +142,7 @@ def _load_remote_code_configs(model_name: str, hf_token: Optional[str] = None) - for name in _REMOTE_CODE_CONFIG_FILES: p = root / name if p.is_file(): - configs.append(json.loads(p.read_text(encoding = "utf-8"))) + configs.append(json.loads(p.read_text(encoding = "utf-8-sig"))) return configs from huggingface_hub import hf_hub_download @@ -164,7 +164,7 @@ def _load_remote_code_configs(model_name: str, hf_token: Optional[str] = None) - # Transient/auth failure is not "absent" -> fail closed to "unknown" so # the caller scans (a tokenizer/processor-only auto_map must not slip by). return None - configs.append(json.loads(Path(p).read_text(encoding = "utf-8"))) + configs.append(json.loads(Path(p).read_text(encoding = "utf-8-sig"))) # Every config was read or a genuine 404 -> an empty list is a definitive # "no auto_map", not "unknown". return configs diff --git a/studio/backend/utils/security/file_security.py b/studio/backend/utils/security/file_security.py index 7724406e8d..4588f32b90 100644 --- a/studio/backend/utils/security/file_security.py +++ b/studio/backend/utils/security/file_security.py @@ -199,7 +199,7 @@ def _indexed_shard_paths( inconclusive = True # transient: an index that might exist could not be read continue try: - weight_map = (json.loads(open(index_path, encoding = "utf-8").read()) or {}).get( + weight_map = (json.loads(open(index_path, encoding = "utf-8-sig").read()) or {}).get( "weight_map" ) or {} for shard in weight_map.values(): @@ -328,7 +328,7 @@ def _st_load_roots(snapshot: Path) -> list: roots = [snapshot] try: import json - modules = json.loads((snapshot / "modules.json").read_text(encoding = "utf-8")) + modules = json.loads((snapshot / "modules.json").read_text(encoding = "utf-8-sig")) except (OSError, ValueError): return roots # no / invalid modules.json -> snapshot root is the only load root for module in modules or (): @@ -355,7 +355,7 @@ def _indexed_pickle_shards(index_path: Path, root: Path, snapshot: Path) -> list try: # JSON is UTF-8 by spec; pin it so a non-ASCII index is not misdecoded (and needlessly # blocked) under Windows' cp1252 default. - parsed = json.loads(index_path.read_text(encoding = "utf-8")) + parsed = json.loads(index_path.read_text(encoding = "utf-8-sig")) except (OSError, ValueError) as exc: raise OSError(f"unreadable weight index: {index_path}") from exc weight_map = parsed.get("weight_map") if isinstance(parsed, dict) else None diff --git a/studio/backend/utils/security/remote_code_approvals.py b/studio/backend/utils/security/remote_code_approvals.py index f1baac6924..d6076fd2b7 100644 --- a/studio/backend/utils/security/remote_code_approvals.py +++ b/studio/backend/utils/security/remote_code_approvals.py @@ -69,7 +69,7 @@ def approval_target_key(targets) -> str: def _load() -> dict: """Parsed store, or an empty skeleton on any error (fail-safe = re-prompt).""" try: - with open(_store_path(), encoding = "utf-8") as f: + with open(_store_path(), encoding = "utf-8-sig") as f: data = json.load(f) # Validate the shape, not just the version: a hand-edited ``subjects`` that is not a # dict (e.g. ``[]``) would otherwise crash lookup/record instead of failing safe. diff --git a/studio/backend/utils/security/remote_code_scan.py b/studio/backend/utils/security/remote_code_scan.py index d4d8003252..42f9d98efe 100644 --- a/studio/backend/utils/security/remote_code_scan.py +++ b/studio/backend/utils/security/remote_code_scan.py @@ -454,7 +454,7 @@ def repo_remote_code_files(model_name: str, hf_token: Optional[str] = None) -> d p = root / name if p.is_file(): try: - ext_refs |= _auto_map_refs(json.loads(p.read_text(encoding = "utf-8"))) + ext_refs |= _auto_map_refs(json.loads(p.read_text(encoding = "utf-8-sig"))) except Exception: pass if not _add_external_refs(files, ext_refs, hf_token, model_name): @@ -483,7 +483,7 @@ def repo_remote_code_files(model_name: str, hf_token: Optional[str] = None) -> d f"{model_name}: config {cfg_name} could not be fetched ({exc})" ) from exc try: - refs |= _auto_map_refs(json.loads(Path(cfg_path).read_text(encoding = "utf-8"))) + refs |= _auto_map_refs(json.loads(Path(cfg_path).read_text(encoding = "utf-8-sig"))) except Exception: pass own_refs = {fn for repo, fn in refs if repo is None} @@ -616,7 +616,7 @@ def external_auto_map_repos(model_name: str, hf_token: Optional[str] = None) -> if not p.is_file(): continue try: - refs = _auto_map_refs(json.loads(p.read_text(encoding = "utf-8"))) + refs = _auto_map_refs(json.loads(p.read_text(encoding = "utf-8-sig"))) except Exception: continue repos.update(repo for repo, _fn in refs if repo) @@ -638,7 +638,7 @@ def external_auto_map_repos(model_name: str, hf_token: Optional[str] = None) -> except Exception: continue try: - refs = _auto_map_refs(json.loads(Path(cfg_path).read_text(encoding = "utf-8"))) + refs = _auto_map_refs(json.loads(Path(cfg_path).read_text(encoding = "utf-8-sig"))) except Exception: continue repos.update(repo for repo, _fn in refs if repo) diff --git a/studio/backend/utils/ssm_runtime.py b/studio/backend/utils/ssm_runtime.py index ca7e2309f9..b864e78608 100644 --- a/studio/backend/utils/ssm_runtime.py +++ b/studio/backend/utils/ssm_runtime.py @@ -23,6 +23,7 @@ import threading from typing import Any, Callable, Optional from loggers import get_logger +from utils.child_stdio import utf8_child_env from utils.wheel_utils import ( direct_wheel_url, install_wheel, @@ -254,6 +255,12 @@ def _install_kernel( "stdout": subprocess.PIPE, "stderr": subprocess.STDOUT, "text": True, + # pip and the compilers it drives write UTF-8 down this pipe; the Windows + # ANSI codepage would mojibake or raise over a fine install. + "encoding": "utf-8", + "errors": "replace", + # Make the Python child emit the UTF-8 we decode above. + "env": utf8_child_env(), } if is_hip: run_kwargs["timeout"] = 1800 # ROCm builds can take 10-30 min @@ -261,7 +268,8 @@ def _install_kernel( if "--gcc-install-dir" not in existing: gcc_dir = _hipcc_gcc_install_dir() if gcc_dir: - _env = os.environ.copy() + # Extends the UTF-8 env above rather than replacing it. + _env = dict(run_kwargs["env"]) _env["HIPCC_COMPILE_FLAGS_APPEND"] = ( f"{existing} --gcc-install-dir={gcc_dir}".strip() ) diff --git a/studio/backend/utils/studio_version.py b/studio/backend/utils/studio_version.py index 82ade74bba..cfaba36a81 100644 --- a/studio/backend/utils/studio_version.py +++ b/studio/backend/utils/studio_version.py @@ -60,6 +60,8 @@ def _exact_git_studio_tag(repo_root: Path) -> str | None: stdout = subprocess.PIPE, stderr = subprocess.DEVNULL, text = True, + encoding = "utf-8", + errors = "replace", timeout = _GIT_TIMEOUT_SECONDS, ) except (OSError, subprocess.TimeoutExpired): @@ -81,6 +83,8 @@ def _git_branch(repo_root: Path) -> str | None: stdout = subprocess.PIPE, stderr = subprocess.DEVNULL, text = True, + encoding = "utf-8", + errors = "replace", timeout = _GIT_TIMEOUT_SECONDS, ) except (OSError, subprocess.TimeoutExpired): diff --git a/studio/backend/utils/transformers_version.py b/studio/backend/utils/transformers_version.py index b0a2da0e66..3774409009 100644 --- a/studio/backend/utils/transformers_version.py +++ b/studio/backend/utils/transformers_version.py @@ -44,6 +44,7 @@ import time from pathlib import Path from utils.native_path_leases import child_env_without_native_path_secret +from utils.child_stdio import utf8_child_env from utils.hf_cache_settings import get_hf_cache_paths from utils.subprocess_compat import ( windows_hidden_subprocess_kwargs as _windows_hidden_subprocess_kwargs, @@ -420,7 +421,7 @@ def _resolve_base_model(model_name: str) -> str: adapter_cfg_path = local_path / "adapter_config.json" if _safe_is_file(adapter_cfg_path): try: - with open(adapter_cfg_path, encoding = "utf-8") as f: + with open(adapter_cfg_path, encoding = "utf-8-sig") as f: cfg = json.load(f) base = cfg.get("base_model_name_or_path") if base: @@ -437,7 +438,7 @@ def _resolve_base_model(model_name: str) -> str: config_json_path = local_path / "config.json" if _safe_is_file(config_json_path): try: - with open(config_json_path, encoding = "utf-8") as f: + with open(config_json_path, encoding = "utf-8-sig") as f: cfg = json.load(f) # Unsloth writes model_name, HF writes _name_or_path; skip a self-reference. for _key in ("model_name", "_name_or_path"): @@ -544,7 +545,7 @@ def _adapter_base_from_hf_cache(model_name: str) -> str | None: ) for cfg_path in candidates: if cfg_path.is_file(): - base = json.loads(cfg_path.read_text(encoding = "utf-8")).get( + base = json.loads(cfg_path.read_text(encoding = "utf-8-sig")).get( "base_model_name_or_path" ) return base or None @@ -616,7 +617,7 @@ def _check_tokenizer_config_needs_v5(model_name: str, hf_token: str | None = Non local_tc = local_path / "tokenizer_config.json" if _safe_is_file(local_tc): try: - with open(local_tc, encoding = "utf-8") as f: + with open(local_tc, encoding = "utf-8-sig") as f: data = json.load(f) tokenizer_class = data.get("tokenizer_class", "") result = tokenizer_class in _TRANSFORMERS_5_TOKENIZER_CLASSES @@ -706,7 +707,7 @@ def _config_json_from_hf_cache(model_name: str) -> dict | None: ) for cfg_path in candidates: if cfg_path.is_file(): - with open(cfg_path, encoding = "utf-8") as f: + with open(cfg_path, encoding = "utf-8-sig") as f: return json.load(f) except Exception as exc: logger.debug("HF cache config.json lookup failed for '%s': %s", model_name, exc) @@ -731,7 +732,7 @@ def _load_config_json(model_name: str, hf_token: str | None = None) -> dict | No local_cfg = Path(model_name) / "config.json" if _safe_is_file(local_cfg): try: - with open(local_cfg, encoding = "utf-8") as f: + with open(local_cfg, encoding = "utf-8-sig") as f: cfg = json.load(f) _config_json_cache[cache_key] = cfg return cfg @@ -1271,9 +1272,10 @@ def _probe_autoconfig(target_dir: str, model_name: str, hf_token: str | None) -> [sys.executable, "-c", _PROBE_CONFIG_SCRIPT, target_dir, model_name], capture_output = True, text = True, + encoding = "utf-8", errors = "replace", timeout = _PROBE_TIMEOUT_SECS, - env = env, + env = utf8_child_env(env), **_windows_hidden_subprocess_kwargs(), ) except subprocess.TimeoutExpired: @@ -1811,7 +1813,11 @@ def _install_to_dir(pkg: str, target_dir: str) -> bool: stdout = subprocess.PIPE, stderr = subprocess.STDOUT, text = True, - env = get_hf_cache_paths().child_env(child_env_without_native_path_secret()), + encoding = "utf-8", + errors = "replace", + env = utf8_child_env( + get_hf_cache_paths().child_env(child_env_without_native_path_secret()) + ), **_windows_hidden_subprocess_kwargs(), ) if result.returncode == 0: @@ -1834,7 +1840,9 @@ def _install_to_dir(pkg: str, target_dir: str) -> bool: stdout = subprocess.PIPE, stderr = subprocess.STDOUT, text = True, - env = get_hf_cache_paths().child_env(child_env_without_native_path_secret()), + encoding = "utf-8", + errors = "replace", + env = utf8_child_env(get_hf_cache_paths().child_env(child_env_without_native_path_secret())), **_windows_hidden_subprocess_kwargs(), ) if result.returncode != 0: @@ -2079,7 +2087,7 @@ class SidecarSwapInProgress(RuntimeError): def _read_swap_lock(path: Path) -> dict | None: try: - data = json.loads(path.read_text(encoding = "utf-8")) + data = json.loads(path.read_text(encoding = "utf-8-sig")) return data if isinstance(data, dict) else {} except FileNotFoundError: return None @@ -2120,7 +2128,7 @@ def try_begin_sidecar_swap(kind: str = "install") -> bool: break if fd is not None: try: - with os.fdopen(fd, "w") as f: + with os.fdopen(fd, "w", encoding = "utf-8") as f: f.write( json.dumps( {"pid": os.getpid(), "at": time.time(), "token": token, "kind": kind} @@ -2466,7 +2474,11 @@ def _ensure_venv_llmcompressor_exists() -> bool: stdout = subprocess.PIPE, stderr = subprocess.STDOUT, text = True, - env = get_hf_cache_paths().child_env(child_env_without_native_path_secret()), + encoding = "utf-8", + errors = "replace", + env = utf8_child_env( + get_hf_cache_paths().child_env(child_env_without_native_path_secret()) + ), **_windows_hidden_subprocess_kwargs(), ) last_out = result.stdout or "" diff --git a/studio/backend/utils/utils.py b/studio/backend/utils/utils.py index e4964b8d04..e830ea2700 100644 --- a/studio/backend/utils/utils.py +++ b/studio/backend/utils/utils.py @@ -114,6 +114,8 @@ def hf_cache_snapshot_dir(model_name: str) -> Optional[Path]: snapshot = repo_dir / "snapshots" / commit if snapshot.is_dir(): return snapshot + # UnicodeDecodeError is a ValueError, not an OSError: a torn refs + # file must keep meaning "not cached here", not fail the offline check. except (OSError, UnicodeDecodeError): continue return None diff --git a/studio/backend/utils/wheel_utils.py b/studio/backend/utils/wheel_utils.py index 1b5926fd49..8ebdea3ac1 100644 --- a/studio/backend/utils/wheel_utils.py +++ b/studio/backend/utils/wheel_utils.py @@ -15,6 +15,7 @@ import urllib.request from typing import Callable from utils.native_path_leases import child_env_without_native_path_secret +from utils.child_stdio import utf8_child_env from utils.subprocess_compat import windows_hidden_subprocess_kwargs _logger = logging.getLogger(__name__) @@ -43,6 +44,8 @@ def has_blackwell_gpu() -> bool: stdout = subprocess.PIPE, stderr = subprocess.DEVNULL, text = True, + encoding = "utf-8", + errors = "replace", timeout = 10, env = child_env_without_native_path_secret(), ) @@ -102,8 +105,10 @@ def probe_torch_wheel_env(*, timeout: int | None = None) -> dict[str, str] | Non stdout = subprocess.PIPE, stderr = subprocess.PIPE, text = True, + encoding = "utf-8", + errors = "replace", timeout = timeout, - env = child_env_without_native_path_secret(), + env = utf8_child_env(child_env_without_native_path_secret()), **windows_hidden_subprocess_kwargs(), ) except subprocess.TimeoutExpired: @@ -201,6 +206,8 @@ def install_wheel( stdout = subprocess.PIPE, stderr = subprocess.STDOUT, text = True, + encoding = "utf-8", + errors = "replace", env = child_env_without_native_path_secret(), ) attempts.append(("uv", result)) @@ -213,7 +220,10 @@ def install_wheel( stdout = subprocess.PIPE, stderr = subprocess.STDOUT, text = True, - env = child_env_without_native_path_secret(), + encoding = "utf-8", + errors = "replace", + # Make the Python child emit the UTF-8 we decode above. + env = utf8_child_env(child_env_without_native_path_secret()), ) attempts.append(("pip", result)) return attempts diff --git a/studio/backend/utils/whisper_cpp_update.py b/studio/backend/utils/whisper_cpp_update.py index cac37c25fc..45a0faf674 100644 --- a/studio/backend/utils/whisper_cpp_update.py +++ b/studio/backend/utils/whisper_cpp_update.py @@ -121,7 +121,14 @@ def _installed_whisper_version(binary: Optional[str]) -> Optional[str]: if not binary: return None try: - proc = subprocess.run([binary, "--version"], capture_output = True, text = True, timeout = 20) + proc = subprocess.run( + [binary, "--version"], + capture_output = True, + text = True, + encoding = "utf-8", + errors = "replace", + timeout = 20, + ) except Exception: # pragma: no cover - defensive return None m = re.search(r"v?(\d+\.\d+\.\d+)", (proc.stderr or "") + (proc.stdout or ""))