From 0f2fc1907669df51bedc8fa4228b4d5e19c7597b Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 14 Aug 2025 15:24:25 -0700 Subject: [PATCH] Encoding UTF-8 --- unsloth/__init__.py | 2 +- unsloth/dataprep/synthetic.py | 6 +++--- unsloth/models/llama.py | 2 +- unsloth/save.py | 8 ++++---- unsloth/tokenizer_utils.py | 8 ++++---- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/unsloth/__init__.py b/unsloth/__init__.py index f856b64f50..5d9ddbd43f 100644 --- a/unsloth/__init__.py +++ b/unsloth/__init__.py @@ -117,7 +117,7 @@ try: cutlass = Path(xformers_location) / "ops" / "fmha" / "cutlass.py" if cutlass.exists(): - with open(cutlass, "r+") as f: + with open(cutlass, "r+", encoding = "utf-8") as f: text = f.read() # See https://github.com/facebookresearch/xformers/issues/1176#issuecomment-2545829591 if "num_splits_key=-1," in text: diff --git a/unsloth/dataprep/synthetic.py b/unsloth/dataprep/synthetic.py index 23268807b1..52c114fab6 100644 --- a/unsloth/dataprep/synthetic.py +++ b/unsloth/dataprep/synthetic.py @@ -230,7 +230,7 @@ class SyntheticDataKit: if not hasattr(self, "overlap") or not hasattr(self, "max_generation_tokens"): raise RuntimeError("Please use prepare_qa_generation first!") - with open(filename, "r") as f: text = f.read() + with open(filename, "r", encoding = "utf-8") as f: text = f.read() max_tokens = self.max_seq_length - self.max_generation_tokens*2 - 128 # -128 to reduce errors if max_tokens <= 5: @@ -253,7 +253,7 @@ class SyntheticDataKit: chunked_text = self.tokenizer.decode(input_ids[left : right]) new_filename = f"{filename}_{i}{extension}" all_filenames.append(new_filename) - with open(new_filename, "w") as f: f.write(chunked_text) + with open(new_filename, "w", encoding = "utf-8") as f: f.write(chunked_text) pass return all_filenames pass @@ -295,7 +295,7 @@ class SyntheticDataKit: .replace("{cleanup_batch_size}", str(cleanup_batch_size))\ .replace("{cleanup_temperature}", str(cleanup_temperature)) - with open("synthetic_data_kit_config.yaml", "w") as f: f.write(config) + with open("synthetic_data_kit_config.yaml", "w", encoding = "utf-8") as f: f.write(config) self.overlap = overlap pass diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index eafbd5a433..ab7f4bfdde 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -1921,7 +1921,7 @@ class FastLlamaModel: has_rope_scaling = False try: - with open(inspect.getfile(model_function), "r") as file: + with open(inspect.getfile(model_function), "r", encoding = "utf-8") as file: has_rope_scaling = "self.config.rope_scaling" in file.read() except: pass has_rope_scaling = True diff --git a/unsloth/save.py b/unsloth/save.py index 38c73c9956..0317c4856c 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -1530,7 +1530,7 @@ def upload_to_huggingface( # We also upload a config.json file if create_config: import json - with open("_temporary_unsloth_config.json", "w") as file: + with open("_temporary_unsloth_config.json", "w", encoding = "utf-8") as file: json.dump({"model_type" : model.config.model_type}, file, indent = 4) pass hf_api.upload_file( @@ -1708,7 +1708,7 @@ def push_to_ollama( gguf_location=gguf_location ) - with open(f"Modelfile_{model_name}", "w") as f: + with open(f"Modelfile_{model_name}", "w", encoding = "utf-8") as f: f.write(model_file) f.close() @@ -1872,7 +1872,7 @@ def unsloth_save_pretrained_gguf( modelfile_location = None if modelfile is not None: modelfile_location = os.path.join(new_save_directory, "Modelfile") - with open(modelfile_location, "w") as file: + with open(modelfile_location, "w", encoding = "utf-8") as file: file.write(modelfile) pass print(f"Unsloth: Saved Ollama Modelfile to {modelfile_location}") @@ -2050,7 +2050,7 @@ def unsloth_push_to_hub_gguf( modelfile_location = None if modelfile is not None: modelfile_location = os.path.join(new_save_directory, "Modelfile") - with open(modelfile_location, "w") as file: + with open(modelfile_location, "w", encoding = "utf-8") as file: file.write(modelfile) pass print(f"Unsloth: Saved Ollama Modelfile to {modelfile_location}") diff --git a/unsloth/tokenizer_utils.py b/unsloth/tokenizer_utils.py index 067f2596c6..8c462cf5d8 100644 --- a/unsloth/tokenizer_utils.py +++ b/unsloth/tokenizer_utils.py @@ -360,7 +360,7 @@ def fix_sentencepiece_tokenizer( old_tokenizer.save_pretrained(temporary_location) tokenizer_file = sentencepiece_model_pb2.ModelProto() - tokenizer_file.ParseFromString(open(f"{temporary_location}/tokenizer.model", "rb").read()) + tokenizer_file.ParseFromString(open(f"{temporary_location}/tokenizer.model", "rb", encoding = "utf-8").read()) # Now save the new tokenizer new_tokenizer.save_pretrained(temporary_location) @@ -385,7 +385,7 @@ def fix_sentencepiece_tokenizer( pass # And now write it - with open(f"{temporary_location}/tokenizer.model", "wb") as file: + with open(f"{temporary_location}/tokenizer.model", "wb", encoding = "utf-8") as file: file.write(tokenizer_file.SerializeToString()) pass @@ -423,7 +423,7 @@ def fix_sentencepiece_gguf(saved_location): # Load tokenizer.model tokenizer_file = sentencepiece_model_pb2.ModelProto() if not os.path.isfile(f"{saved_location}/tokenizer.model"): return - tokenizer_file.ParseFromString(open(f"{saved_location}/tokenizer.model", "rb").read()) + tokenizer_file.ParseFromString(open(f"{saved_location}/tokenizer.model", "rb", encoding = "utf-8").read()) sentence_piece_size = len(tokenizer_file.pieces) # Load added_tokens_json @@ -457,7 +457,7 @@ def fix_sentencepiece_gguf(saved_location): tokenizer_file.pieces.extend(new_tokens) - with open(f"{saved_location}/tokenizer.model", "wb") as file: + with open(f"{saved_location}/tokenizer.model", "wb", encoding = "utf-8") as file: file.write(tokenizer_file.SerializeToString()) pass