Merge branch 'main' into nightly

unsloth/chat_templates.py

@@ -2593,7 +2593,7 @@ default_system_message = \
 extra_eos_tokens = None,
 ):
     """
-    Creates a Ollama modelfile and a HF Jinja template from a custom
+    Creates an Ollama modelfile and a HF Jinja template from a custom
     template. You must provide 2x examples of an input & output.
     There is an optional system message as well.
 
@@ -2930,7 +2930,7 @@ extra_eos_tokens = None,
 
 ):
     """
-    Creates a Ollama modelfile and a HF Jinja template from a custom
+    Creates an Ollama modelfile and a HF Jinja template from a custom
     template. You must provide 2x examples of an input & output.
     There is an optional system message as well.
 

unsloth/models/falcon_h1.py

@@ -57,7 +57,7 @@ try:
         FalconH1Attention,
     )
 except ModuleNotFoundError:
-    # if we are on a old version of transformers technically it should fail in the try except above
+    # if we are on an old version of transformers technically it should fail in the try except above
     # but if somehow we make it here, we need to raise an error since FalconH1Attention is not available
     # or renamed
     raise ImportError(

unsloth/models/llama.py

@@ -1988,6 +1988,9 @@ def unsloth_fast_generate(
     *args,
     **kwargs,
 ):
+    # If the model starts out in training mode, restore training mode after generation
+    restore_training_mode = self.training
+
     FastLlamaModel.for_inference(self)
 
     dtype = _get_dtype(dtype_from_config(self.config))
@@ -2043,7 +2046,8 @@ def unsloth_fast_generate(
     #     accelerate.utils.operations.send_to_device = accelerate_old_send_to_device
     # pass
 
-    FastLlamaModel.for_training(self)
+    if restore_training_mode:
+        FastLlamaModel.for_training(self)
 
     return output
 

unsloth/models/loader.py

@@ -258,7 +258,7 @@ class FastLanguageModel(FastLlamaModel):
         if model_name.lower().endswith("-bf16"):
             load_in_4bit = False
             load_in_8bit = False
-            load_in_fp8  = False
+            load_in_fp8 = False
             load_in_16bit = True
 
         if USE_MODELSCOPE and not os.path.exists(model_name):
@@ -387,7 +387,7 @@ class FastLanguageModel(FastLlamaModel):
             if model_name.lower().endswith("-bf16"):
                 load_in_4bit = False
                 load_in_8bit = False
-                load_in_fp8  = False
+                load_in_fp8 = False
                 load_in_16bit = True
 
             model_config = AutoConfig.from_pretrained(
@@ -711,10 +711,16 @@ class FastModel(FastBaseModel):
             )
             load_in_4bit = False
             load_in_8bit = False
-            load_in_fp8  = False
+            load_in_fp8 = False
             load_in_16bit = False
 
-        if int(load_in_4bit) + int(load_in_8bit) + int(load_in_16bit) + int(load_in_fp8 != False) >= 2:
+        if (
+            int(load_in_4bit)
+            + int(load_in_8bit)
+            + int(load_in_16bit)
+            + int(load_in_fp8 != False)
+            >= 2
+        ):
             raise RuntimeError(
                 "Unsloth: Can only load in 4bit or 8bit or 16bit, not a combination!\n"
                 "Also, we by default set `load_in_4bit = True`.\n"

unsloth/ollama_template_mappers.py

@@ -1520,7 +1520,7 @@ Loop over messages and look for a user-provided system message and documents
 
     {{- /*
     NOTE: Since Ollama collates consecutive roles, for control and documents, we
-        work around this by allowing the role to contain an qualifier after the
+        work around this by allowing the role to contain a qualifier after the
         role string.
     */ -}}
 

unsloth/tokenizer_utils.py

@@ -600,7 +600,7 @@ def load_correct_tokenizer(
     ### 1. Fixup tokenizer's chat_template
     old_chat_template = getattr(tokenizer, "chat_template", None)
 
-    # Ignore mistral type models since they don't have a add_generation_prompt
+    # Ignore mistral type models since they don't have an add_generation_prompt
     if "mistral" in str(getattr(tokenizer, "name_or_path", "")).lower():
         chat_template = old_chat_template