Merge branch 'main' into nightly

Author: danielhanchen

unsloth/__init__.py

@@ -117,7 +117,7 @@ def get_device_count():
         cutlass = Path(xformers_location) / "ops" / "fmha" / "cutlass.py"
 
         if cutlass.exists():
-            with open(cutlass, "r+") as f:
+            with open(cutlass, "r+", encoding = "utf-8") as f:
                 text = f.read()
                 # See https://github.com/facebookresearch/xformers/issues/1176#issuecomment-2545829591
                 if "num_splits_key=-1," in text:

unsloth/dataprep/synthetic.py

@@ -230,7 +230,7 @@ def chunk_data(self, filename = None):
         if not hasattr(self, "overlap") or not hasattr(self, "max_generation_tokens"):
             raise RuntimeError("Please use prepare_qa_generation first!")
 
-        with open(filename, "r") as f: text = f.read()
+        with open(filename, "r", encoding = "utf-8") as f: text = f.read()
 
         max_tokens = self.max_seq_length - self.max_generation_tokens*2 - 128 # -128 to reduce errors
         if max_tokens <= 5:
@@ -253,7 +253,7 @@ def chunk_data(self, filename = None):
             chunked_text = self.tokenizer.decode(input_ids[left : right])
             new_filename = f"{filename}_{i}{extension}"
             all_filenames.append(new_filename)
-            with open(new_filename, "w") as f: f.write(chunked_text)
+            with open(new_filename, "w", encoding = "utf-8") as f: f.write(chunked_text)
         pass
         return all_filenames
     pass
@@ -295,7 +295,7 @@ def prepare_qa_generation(
             .replace("{cleanup_batch_size}", str(cleanup_batch_size))\
             .replace("{cleanup_temperature}", str(cleanup_temperature))
 
-        with open("synthetic_data_kit_config.yaml", "w") as f: f.write(config)
+        with open("synthetic_data_kit_config.yaml", "w", encoding = "utf-8") as f: f.write(config)
 
         self.overlap = overlap
     pass

unsloth/models/llama.py

@@ -1921,7 +1921,7 @@ def from_pretrained(
 
         has_rope_scaling = False
         try:
-            with open(inspect.getfile(model_function), "r") as file:
+            with open(inspect.getfile(model_function), "r", encoding = "utf-8") as file:
                 has_rope_scaling = "self.config.rope_scaling" in file.read()
         except: pass
         has_rope_scaling = True

unsloth/save.py

@@ -1530,7 +1530,7 @@ def upload_to_huggingface(
         # We also upload a config.json file
         if create_config:
             import json
-            with open("_temporary_unsloth_config.json", "w") as file:
+            with open("_temporary_unsloth_config.json", "w", encoding = "utf-8") as file:
                 json.dump({"model_type" : model.config.model_type}, file, indent = 4)
             pass
             hf_api.upload_file(
@@ -1708,7 +1708,7 @@ def push_to_ollama(
         gguf_location=gguf_location
     )
 
-    with open(f"Modelfile_{model_name}", "w") as f:
+    with open(f"Modelfile_{model_name}", "w", encoding = "utf-8") as f:
         f.write(model_file)
         f.close()
 
@@ -1872,7 +1872,7 @@ def unsloth_save_pretrained_gguf(
     modelfile_location = None
     if modelfile is not None:
         modelfile_location = os.path.join(new_save_directory, "Modelfile")
-        with open(modelfile_location, "w") as file:
+        with open(modelfile_location, "w", encoding = "utf-8") as file:
             file.write(modelfile)
         pass
         print(f"Unsloth: Saved Ollama Modelfile to {modelfile_location}")
@@ -2050,7 +2050,7 @@ def unsloth_push_to_hub_gguf(
     modelfile_location = None
     if modelfile is not None:
         modelfile_location = os.path.join(new_save_directory, "Modelfile")
-        with open(modelfile_location, "w") as file:
+        with open(modelfile_location, "w", encoding = "utf-8") as file:
             file.write(modelfile)
         pass
         print(f"Unsloth: Saved Ollama Modelfile to {modelfile_location}")

unsloth/tokenizer_utils.py

@@ -360,7 +360,7 @@ def fix_sentencepiece_tokenizer(
     old_tokenizer.save_pretrained(temporary_location)
 
     tokenizer_file = sentencepiece_model_pb2.ModelProto()
-    tokenizer_file.ParseFromString(open(f"{temporary_location}/tokenizer.model", "rb").read())
+    tokenizer_file.ParseFromString(open(f"{temporary_location}/tokenizer.model", "rb", encoding = "utf-8").read())
 
     # Now save the new tokenizer
     new_tokenizer.save_pretrained(temporary_location)
@@ -385,7 +385,7 @@ def fix_sentencepiece_tokenizer(
     pass
 
     # And now write it
-    with open(f"{temporary_location}/tokenizer.model", "wb") as file:
+    with open(f"{temporary_location}/tokenizer.model", "wb", encoding = "utf-8") as file:
         file.write(tokenizer_file.SerializeToString())
     pass
 
@@ -423,7 +423,7 @@ class SentencePieceTokenTypes(IntEnum):
     # Load tokenizer.model
     tokenizer_file = sentencepiece_model_pb2.ModelProto()
     if not os.path.isfile(f"{saved_location}/tokenizer.model"): return
-    tokenizer_file.ParseFromString(open(f"{saved_location}/tokenizer.model", "rb").read())
+    tokenizer_file.ParseFromString(open(f"{saved_location}/tokenizer.model", "rb", encoding = "utf-8").read())
     sentence_piece_size = len(tokenizer_file.pieces)
 
     # Load added_tokens_json
@@ -457,7 +457,7 @@ class SentencePieceTokenTypes(IntEnum):
 
     tokenizer_file.pieces.extend(new_tokens)
 
-    with open(f"{saved_location}/tokenizer.model", "wb") as file:
+    with open(f"{saved_location}/tokenizer.model", "wb", encoding = "utf-8") as file:
         file.write(tokenizer_file.SerializeToString())
     pass