From 1daaa5cbb4fd5fe13248d918d9a50c55eb091eb6 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 27 Jul 2026 03:26:08 -0700 Subject: [PATCH] Let a decode failure degrade instead of escaping a fail-closed helper (#7487) * Let a decode failure degrade instead of escaping a fail-closed helper Pinning utf-8 makes a read that used to return mojibake on Windows raise instead. 33 of those reads sit under a handler catching OSError or json.JSONDecodeError but not UnicodeDecodeError, which subclasses ValueError, so a corrupt file would now escape a helper written to return a default. Adds UnicodeDecodeError to those tuples only. * Treat an undecodable install lock as stale instead of retrying forever --- studio/backend/colab.py | 2 +- studio/backend/core/inference/llama_cpp.py | 12 ++++++------ studio/backend/hub/services/models/ollama.py | 4 ++-- studio/backend/hub/utils/download_registry.py | 2 +- studio/backend/main.py | 2 +- .../src/data_designer_unstructured_seed/impl.py | 2 +- studio/backend/routes/models.py | 4 ++-- studio/backend/run.py | 4 ++-- studio/backend/utils/hardware/hardware.py | 2 +- studio/backend/utils/paths/storage_roots.py | 2 +- studio/backend/utils/utils.py | 2 +- studio/install_llama_prebuilt.py | 6 +++--- studio/install_node_prebuilt.py | 6 ++++-- studio/install_python_stack.py | 10 +++++----- studio/prebuilt_core.py | 6 ++++-- unsloth_cli/_inference.py | 2 +- unsloth_cli/commands/studio.py | 2 +- 17 files changed, 37 insertions(+), 33 deletions(-) diff --git a/studio/backend/colab.py b/studio/backend/colab.py index df1285b749..bf4a6a44b5 100644 --- a/studio/backend/colab.py +++ b/studio/backend/colab.py @@ -109,7 +109,7 @@ def _load_colab_login_credentials() -> "tuple[str, str] | None": lines = path.read_text(encoding = "utf-8").splitlines() if len(lines) >= 2 and lines[0] and lines[1]: return lines[0], lines[1] - except OSError as e: + except (OSError, UnicodeDecodeError) as e: logger.info(f"Could not load Colab login credentials ({e}).") return None diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index f286a2e4c5..0621a7f9c8 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -246,7 +246,7 @@ def _wsl_system_rocm_lib_dirs() -> "list[str]": with open("/proc/version", encoding = "utf-8", errors = "replace") as fh: if "microsoft" not in fh.read().lower(): return [] - except OSError: + except (OSError, UnicodeDecodeError): return [] out: "list[str]" = [] for d in ("/opt/rocm/lib", "/opt/rocm/lib64"): @@ -573,7 +573,7 @@ def _load_swa_cache() -> dict: _SWA_CACHE = json.load(f) if not isinstance(_SWA_CACHE, dict): _SWA_CACHE = {} - except (FileNotFoundError, json.JSONDecodeError, OSError): + except (FileNotFoundError, json.JSONDecodeError, OSError, UnicodeDecodeError): _SWA_CACHE = {} return _SWA_CACHE @@ -586,7 +586,7 @@ def _save_swa_cache(cache: dict) -> None: with open(tmp, "w", encoding = "utf-8") as f: json.dump(cache, f, indent = 2, sort_keys = True) tmp.replace(path) - except OSError: + except (OSError, UnicodeDecodeError): pass @@ -5138,7 +5138,7 @@ class LlamaCppBackend: self._llama_log_path = log_dir / f"diffusion-{int(time.time())}-port-{self._port}.log" self._llama_log_fh = open(self._llama_log_path, "w", encoding = "utf-8", buffering = 1) logger.info(f"diffusion runner stdout/stderr -> {self._llama_log_path}") - except OSError as e: + except (OSError, UnicodeDecodeError) as e: logger.debug(f"Could not open diffusion runner log file: {e}") # The shim (and its visual server) die with this backend process, so a @@ -6355,7 +6355,7 @@ class LlamaCppBackend: buffering = 1, ) logger.info(f"llama-server stdout/stderr -> {self._llama_log_path}") - except OSError as e: + except (OSError, UnicodeDecodeError) as e: # Best-effort; never block the load on logging. logger.debug(f"Could not open llama-server log file: {e}") self._llama_log_path = None @@ -8302,7 +8302,7 @@ class LlamaCppBackend: buffering = 1, ) logger.info(f"llama-server stdout/stderr -> {self._llama_log_path}") - except OSError as e: + except (OSError, UnicodeDecodeError) as e: # Best-effort; never block the load on logging. logger.debug(f"Could not open llama-server log file: {e}") self._llama_log_path = None diff --git a/studio/backend/hub/services/models/ollama.py b/studio/backend/hub/services/models/ollama.py index 190aef0c71..56275c22a9 100644 --- a/studio/backend/hub/services/models/ollama.py +++ b/studio/backend/hub/services/models/ollama.py @@ -216,7 +216,7 @@ def _ollama_model_info_from_manifest( try: manifest = json.loads(tag_file.read_text(encoding = "utf-8")) - except (json.JSONDecodeError, OSError) as e: + except (json.JSONDecodeError, OSError, UnicodeDecodeError) as e: logger.debug("Skipping unreadable/invalid Ollama manifest %s: %s", tag_file, e) return None @@ -231,7 +231,7 @@ def _ollama_model_info_from_manifest( cfg = json.loads(config_blob.read_text(encoding = "utf-8")) model_type = cfg.get("model_type", "") file_type = cfg.get("file_type", "") - except (json.JSONDecodeError, OSError) as e: + except (json.JSONDecodeError, OSError, UnicodeDecodeError) as e: logger.debug("Could not parse Ollama config blob %s: %s", config_blob, e) layers = manifest.get("layers") or [] diff --git a/studio/backend/hub/utils/download_registry.py b/studio/backend/hub/utils/download_registry.py index 243caab8f7..39c27208b1 100644 --- a/studio/backend/hub/utils/download_registry.py +++ b/studio/backend/hub/utils/download_registry.py @@ -463,7 +463,7 @@ def _read_marker_value(marker: Path) -> Optional[str]: if not marker.exists(): return None value = marker.read_text(encoding = "utf-8").strip() - except OSError: + except (OSError, UnicodeDecodeError): return None return value if value in VALID_TRANSPORTS else None diff --git a/studio/backend/main.py b/studio/backend/main.py index bcf5c281df..e632c9525b 100644 --- a/studio/backend/main.py +++ b/studio/backend/main.py @@ -362,7 +362,7 @@ def get_unsloth_version() -> str: for line in version_file.read_text(encoding = "utf-8").splitlines(): if line.startswith("__version__ = "): return line.split("=", 1)[1].strip().strip('"').strip("'") - except OSError: + except (OSError, UnicodeDecodeError): pass return "dev" diff --git a/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/impl.py b/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/impl.py index 6016f5611f..ce0c88e5bf 100644 --- a/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/impl.py +++ b/studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/impl.py @@ -29,7 +29,7 @@ class UnstructuredSeedReader(SeedReader[UnstructuredSeedSource]): try: meta = json_mod.loads(meta_path.read_text(encoding = "utf-8")) orig_name = meta.get("original_filename", path_obj.name) - except (json_mod.JSONDecodeError, OSError): + except (json_mod.JSONDecodeError, OSError, UnicodeDecodeError): pass file_entries.append((path_obj, orig_name)) diff --git a/studio/backend/routes/models.py b/studio/backend/routes/models.py index dc850becf0..96c5b96d73 100644 --- a/studio/backend/routes/models.py +++ b/studio/backend/routes/models.py @@ -723,7 +723,7 @@ def _scan_ollama_dir(ollama_dir: Path, limit: Optional[int] = None) -> List[Loca try: manifest = json.loads(tag_file.read_text(encoding = "utf-8")) - except (json.JSONDecodeError, OSError) as e: + except (json.JSONDecodeError, OSError, UnicodeDecodeError) as e: logger.debug( "Skipping unreadable/invalid Ollama manifest %s: %s", tag_file, @@ -741,7 +741,7 @@ def _scan_ollama_dir(ollama_dir: Path, limit: Optional[int] = None) -> List[Loca cfg = json.loads(config_blob.read_text(encoding = "utf-8")) model_type = cfg.get("model_type", "") file_type = cfg.get("file_type", "") - except (json.JSONDecodeError, OSError) as e: + except (json.JSONDecodeError, OSError, UnicodeDecodeError) as e: logger.debug( "Could not parse Ollama config blob %s: %s", config_blob, diff --git a/studio/backend/run.py b/studio/backend/run.py index 2189388cf9..5dfab9346a 100644 --- a/studio/backend/run.py +++ b/studio/backend/run.py @@ -786,7 +786,7 @@ def _remove_pid_file(): stored = _PID_FILE.read_text(encoding = "utf-8").strip() if stored == str(os.getpid()): _PID_FILE.unlink(missing_ok = True) - except OSError: + except (OSError, UnicodeDecodeError): pass @@ -934,7 +934,7 @@ def _iter_frontend_fallback_candidates() -> "list[Path]": for finder in sp.glob("__editable___*_finder.py"): try: src = finder.read_text(encoding = "utf-8") - except OSError: + except (OSError, UnicodeDecodeError): continue # Tolerate single/multi-line dict literals; [^}]* rejects nested # dicts, which the setuptools editable template never emits. diff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.py index f3b968c8df..b270a8e671 100644 --- a/studio/backend/utils/hardware/hardware.py +++ b/studio/backend/utils/hardware/hardware.py @@ -903,7 +903,7 @@ def _rocm_kfd_gpu_pci_ids() -> list[str]: props[parts[0]] = int(parts[1]) except ValueError: continue - except OSError: + except (OSError, UnicodeDecodeError): return [] # unreadable node could be a GPU: fail closed, don't shift if props.get("simd_count", 0) <= 0: continue # CPU node, not a GPU diff --git a/studio/backend/utils/paths/storage_roots.py b/studio/backend/utils/paths/storage_roots.py index ab888ec49e..ae1319d296 100644 --- a/studio/backend/utils/paths/storage_roots.py +++ b/studio/backend/utils/paths/storage_roots.py @@ -126,7 +126,7 @@ def _xdg_user_dir(key: str) -> Path | None: config = Path.home() / ".config" / "user-dirs.dirs" try: lines = config.read_text(encoding = "utf-8").splitlines() - except OSError: + except (OSError, UnicodeDecodeError): return None prefix = f"{key}=" for line in lines: diff --git a/studio/backend/utils/utils.py b/studio/backend/utils/utils.py index 2d9306f6ed..e4964b8d04 100644 --- a/studio/backend/utils/utils.py +++ b/studio/backend/utils/utils.py @@ -114,7 +114,7 @@ def hf_cache_snapshot_dir(model_name: str) -> Optional[Path]: snapshot = repo_dir / "snapshots" / commit if snapshot.is_dir(): return snapshot - except OSError: + except (OSError, UnicodeDecodeError): continue return None diff --git a/studio/install_llama_prebuilt.py b/studio/install_llama_prebuilt.py index 47ad4bfc66..9b787dbb15 100644 --- a/studio/install_llama_prebuilt.py +++ b/studio/install_llama_prebuilt.py @@ -2508,7 +2508,7 @@ def detect_host() -> HostInfo: if _vf.read().strip().lower() == "0x8086": has_intel_gpu = True break - except OSError: + except (OSError, UnicodeDecodeError): continue elif is_windows: # Registry first (in-process; see windows_intel_gpu_in_registry). @@ -4264,7 +4264,7 @@ def free_local_port() -> int: def read_log_excerpt(log_path: Path, *, max_lines: int = 60) -> str: try: content = log_path.read_text(encoding = "utf-8", errors = "replace") - except FileNotFoundError: + except (FileNotFoundError, UnicodeDecodeError): return "" return "\n".join(content.splitlines()[-max_lines:]) @@ -4680,7 +4680,7 @@ def _wsl_system_rocm_lib_dirs() -> list[str]: with open("/proc/version", encoding = "utf-8", errors = "replace") as fh: if "microsoft" not in fh.read().lower(): return [] - except OSError: + except (OSError, UnicodeDecodeError): return [] out: list[str] = [] for d in ("/opt/rocm/lib", "/opt/rocm/lib64"): diff --git a/studio/install_node_prebuilt.py b/studio/install_node_prebuilt.py index 948743c63c..82ca1d2c68 100644 --- a/studio/install_node_prebuilt.py +++ b/studio/install_node_prebuilt.py @@ -535,7 +535,9 @@ def install_lock(lock_path: Path) -> Iterator[None]: break except FileExistsError: try: - raw = lock_path.read_text(encoding = "utf-8").strip() + # errors="replace" so an undecodable lock reaches the int() + # below and is treated as a stale PID, not retried forever. + raw = lock_path.read_text(encoding = "utf-8", errors = "replace").strip() except FileNotFoundError: continue stale = False @@ -669,7 +671,7 @@ def load_metadata(install_dir: Path) -> dict | None: return None try: data = json.loads(path.read_text(encoding = "utf-8")) - except (json.JSONDecodeError, OSError): + except (json.JSONDecodeError, OSError, UnicodeDecodeError): return None return data if isinstance(data, dict) else None diff --git a/studio/install_python_stack.py b/studio/install_python_stack.py index 5e9adb79d1..a91f26910f 100644 --- a/studio/install_python_stack.py +++ b/studio/install_python_stack.py @@ -776,7 +776,7 @@ def _linux_amd_gfx_from_cpuinfo() -> "str | None": """Infer gfx arch from /proc/cpuinfo on integrated AMD APUs (Strix Halo/Point).""" try: text = Path("/proc/cpuinfo").read_text(encoding = "utf-8", errors = "replace") - except OSError: + except (OSError, UnicodeDecodeError): return None if re.search(r"Ryzen AI Max|Radeon 80[0-9][05]S|Strix Halo", text, re.IGNORECASE): return "gfx1151" @@ -828,7 +828,7 @@ def _is_wsl() -> bool: try: with open("/proc/version", encoding = "utf-8", errors = "replace") as fh: return "microsoft" in fh.read().lower() - except OSError: + except (OSError, UnicodeDecodeError): return False @@ -856,7 +856,7 @@ def _linux_amd_display_device_present() -> bool: continue if (dev / "class").read_text(encoding = "utf-8").strip().startswith("0x03"): return True - except OSError: + except (OSError, UnicodeDecodeError): continue except OSError: pass @@ -1069,7 +1069,7 @@ def _has_rocm_gpu() -> bool: try: with open(gpu_id_path, encoding = "utf-8") as fh: gpu_id = fh.read().strip() - except OSError: + except (OSError, UnicodeDecodeError): continue if not gpu_id or gpu_id == "0": # gpu_id 0 = CPU node continue @@ -1081,7 +1081,7 @@ def _has_rocm_gpu() -> bool: try: with open(props_path, encoding = "utf-8") as fh: props = fh.read() - except OSError: + except (OSError, UnicodeDecodeError): continue # can't confirm vendor -- skip if not re.search(r"\bvendor_id\s+4098\b", props): continue diff --git a/studio/prebuilt_core.py b/studio/prebuilt_core.py index 1f711d14f0..d7075815d3 100644 --- a/studio/prebuilt_core.py +++ b/studio/prebuilt_core.py @@ -1078,7 +1078,9 @@ def install_lock(lock_path: Path) -> Iterator[None]: except FileExistsError: stale = False try: - raw = lock_path.read_text(encoding = "utf-8").strip() + # errors="replace" so an undecodable lock reaches the int() + # below and is treated as a corrupt PID, not retried forever. + raw = lock_path.read_text(encoding = "utf-8", errors = "replace").strip() except FileNotFoundError: # Lock vanished between our open and read -- retry continue @@ -2070,7 +2072,7 @@ def load_prebuilt_metadata(ops: ModuleOps, install_dir: Path) -> dict[str, Any] return None try: payload = json.loads(path.read_text(encoding = "utf-8")) - except (json.JSONDecodeError, OSError): + except (json.JSONDecodeError, OSError, UnicodeDecodeError): return None return payload if isinstance(payload, dict) else None diff --git a/unsloth_cli/_inference.py b/unsloth_cli/_inference.py index 32b1694129..3ad901542c 100644 --- a/unsloth_cli/_inference.py +++ b/unsloth_cli/_inference.py @@ -100,7 +100,7 @@ def _json_rank_count_from_env(name: str) -> Optional[int]: else: with open(value, "r", encoding = "utf-8") as f: data = json.load(f) - except (OSError, json.JSONDecodeError): + except (json.JSONDecodeError, OSError, UnicodeDecodeError): return None if isinstance(data, list): return len(data) diff --git a/unsloth_cli/commands/studio.py b/unsloth_cli/commands/studio.py index 323e713078..d66dfd7e07 100644 --- a/unsloth_cli/commands/studio.py +++ b/unsloth_cli/commands/studio.py @@ -335,7 +335,7 @@ def _iter_editable_studio_source_roots(venv_dir: Path): for finder in sp.glob("__editable___*_finder.py"): try: src = finder.read_text(encoding = "utf-8") - except OSError: + except (OSError, UnicodeDecodeError): continue # Tolerate single- or multi-line dict literals; [^}]* still # rejects nested dicts, which the setuptools template never