Fix: Add missing utf-8 encoding to text-mode file operations (#5356)

Fixes #2795 by explicitly adding encoding='utf-8' to open() calls. This prevents UnicodeDecodeError on Windows with non-UTF-8 system locales when processing files containing UTF-8 characters.

Co-authored-by: Roland Tannous <115670425+rolandtannous@users.noreply.github.com>

unsloth/kernels/moe/autotune_cache.py

@@ -71,7 +71,7 @@ def load_cached_config(cache_key: str) -> Optional[Dict[str, Any]]:
         return None
 
     try:
-        with open(cache_file, "r") as f:
+        with open(cache_file, "r", encoding = "utf-8") as f:
             cached_data = json.load(f)
 
         # Verify cache is still valid (same device, etc.)
@@ -118,7 +118,7 @@ def save_cached_config(
     }
 
     try:
-        with open(cache_file, "w") as f:
+        with open(cache_file, "w", encoding = "utf-8") as f:
             json.dump(cache_data, f, indent = 2)
         logger.info(f"Saved MoE kernel config cache: {cache_key}")
     except Exception as e:

unsloth/kernels/moe/benchmark/utils.py

@@ -183,7 +183,7 @@ def save_autotune_results(autotune_cache, mode, ref_time, fused_time, results_di
         filename = "_".join(key)
         save_path = f"{save_dir}/{filename}.json"
         print(f"Saving autotune results to {save_path}")
-        with open(save_path, "w") as f:
+        with open(save_path, "w", encoding = "utf-8") as f:
             result = {
                 **config.all_kwargs(),
                 "ref_time": ref_time,

unsloth/models/sentence_transformer.py

@@ -70,7 +70,7 @@ def _save_pretrained_torchao(
     modules_path = os.path.join(save_directory, "modules.json")
     if os.path.exists(modules_path):
         try:
-            with open(modules_path, "r") as f:
+            with open(modules_path, "r", encoding = "utf-8") as f:
                 modules = json.load(f)
             for m in modules:
                 if m.get("type", "").endswith("Transformer"):
@@ -177,7 +177,7 @@ def _save_pretrained_gguf(
     modules_path = os.path.join(save_directory, "modules.json")
     if os.path.exists(modules_path):
         try:
-            with open(modules_path, "r") as f:
+            with open(modules_path, "r", encoding = "utf-8") as f:
                 modules = json.load(f)
             for m in modules:
                 if m.get("type", "").endswith("Transformer"):
@@ -542,7 +542,7 @@ class FastSentenceTransformer(FastModel):
                     model_name, "modules.json", token = token
                 )
 
-            with open(modules_json_path, "r") as f:
+            with open(modules_json_path, "r", encoding = "utf-8") as f:
                 modules_config = json.load(f)
 
             pooling_config_path = None
@@ -566,7 +566,7 @@ class FastSentenceTransformer(FastModel):
                         break
 
             if pooling_config_path:
-                with open(pooling_config_path, "r") as f:
+                with open(pooling_config_path, "r", encoding = "utf-8") as f:
                     pooling_config = json.load(f)
                     # from here:
                     # https://github.com/huggingface/sentence-transformers/blob/main/sentence_transformers/models/Pooling.py#L43

unsloth/save.py

@@ -2641,7 +2641,7 @@ This model was finetuned and converted to GGUF format using [Unsloth](https://gi
         )
 
         readme_path = os.path.join(actual_save_directory, "README.md")
-        with open(readme_path, "w") as f:
+        with open(readme_path, "w", encoding = "utf-8") as f:
             f.write(readme_content)
 
         api.upload_file(