From d8ce12fe5d65b2ac3ef64d4f3ca3ddfd30a6e5fd Mon Sep 17 00:00:00 2001
From: Daniel Han <danielhanchen@gmail.com>
Date: Sat, 24 Feb 2024 17:35:29 +1100
Subject: [PATCH] Updated Home (markdown)

---
 Home.md | 45 +++++++++++++++++++++++++++++----------------
 1 file changed, 29 insertions(+), 16 deletions(-)

diff --git a/Home.md b/Home.md
index 9df827a..207b717 100644
--- a/Home.md
+++ b/Home.md
@@ -55,23 +55,20 @@ model.save_pretrained_merged("model", tokenizer, save_method = "lora",)
 model.push_to_hub_merged("hf/model", tokenizer, save_method = "lora", token = "")
 ```
 
-### Evaluation Loop - also OOM or crashing.
-Set the trainer settings for evaluation to:
-```python
-SFTTrainer(
-    args = TrainingArguments(
-        fp16_full_eval = True,
-        per_device_eval_batch_size = 2,
-        eval_accumulation_steps = 4,
-        evaluation_strategy = "steps",
-        eval_steps = 1,
-    ),
-    train_dataset = train_dataset,
-    eval_dataset = eval_dataset,
-```
-This will cause no OOMs and make it somewhat faster with no upcasting to float32.
+### Saving to GGUF
 
-### GGUF quantization options
+To save to GGUF, use the below to save locally:
+```python
+model.save_pretrained_gguf("dir", tokenizer, quantization_method = "q4_k_m")
+model.save_pretrained_gguf("dir", tokenizer, quantization_method = "q8_0")
+model.save_pretrained_gguf("dir", tokenizer, quantization_method = "f16")
+```
+For to push to hub:
+```python
+model.push_to_hub_gguf("hf_username/dir", tokenizer, quantization_method = "q4_k_m")
+model.push_to_hub_gguf("hf_username/dir", tokenizer, quantization_method = "q8_0")
+```
+All supported quantization options for `quantization_method` are listed below:
 ```python
 # https://github.com/ggerganov/llama.cpp/blob/master/examples/quantize/quantize.cpp#L19
 # From https://mlabonne.github.io/blog/posts/Quantize_Llama_2_models_using_ggml.html
@@ -105,6 +102,22 @@ ALLOWED_QUANTS = \
 }
 ```
 
+### Evaluation Loop - also OOM or crashing.
+Set the trainer settings for evaluation to:
+```python
+SFTTrainer(
+    args = TrainingArguments(
+        fp16_full_eval = True,
+        per_device_eval_batch_size = 2,
+        eval_accumulation_steps = 4,
+        evaluation_strategy = "steps",
+        eval_steps = 1,
+    ),
+    train_dataset = train_dataset,
+    eval_dataset = eval_dataset,
+```
+This will cause no OOMs and make it somewhat faster with no upcasting to float32.
+
 ### Chat Templates
 Assuming your dataset is a list of list of dictionaries like the below:
 ```python