Fix Gemma GGUF (#234)
* Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update llama.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update llama.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * revert * revert * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update llama.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update cross_entropy_loss.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * rope * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * llama * Update llama.py * gemma * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update save.py * RoPE * Update llama.py * Update llama.py * Update llama.py * Update gemma.py * correct_dtype * Update gemma.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Chat Templates * Update README.md * Update README.md * Update llama.py * DoRA * Update _utils.py * Update chat_templates.py * Update llama.py * Hotfix - fix DoRA, Gemma prompt template (#202) (#203) * Update save.py * saving * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update __init__.py * Update save.py * Update save.py * Update save.py * save * trainer * spaces * original * Gemma * Update pyproject.toml * Update mapper.py * Update fast_lora.py * FastGemmaModel * model_type * Update llama.py * Update llama.py * Update gemma.py * Update gemma.py * Update gemma.py * Update llama.py * Update llama.py * Update fast_lora.py * Update llama.py * Update llama.py * Update cross_entropy_loss.py * Update llama.py * Update llama.py * gemma * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update fast_lora.py * Update fast_lora.py * Fast CE Loss * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * CE * Update llama.py * Update llama.py * Update cross_entropy_loss.py * Update geglu.py * Update cross_entropy_loss.py * revert * Update llama.py * Update llama.py * norm * Update gemma.py * Update gemma.py * position_ids * Update gemma.py * Update gemma.py * pos * Update llama.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update cross_entropy_loss.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update llama.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update llama.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * revert * revert * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update llama.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update cross_entropy_loss.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * rope * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * llama * Update llama.py * gemma * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update save.py * RoPE * Update llama.py * Update llama.py * Update llama.py * Update gemma.py * correct_dtype * Update gemma.py * Update cross_entropy_loss.py * Update cross_entropy_loss.py * Chat Templates * Update README.md * Update README.md * Update llama.py * DoRA * Update _utils.py * Update chat_templates.py * Update pyproject.toml * Small fixes * Update pyproject.toml * Approx gelu * Update geglu.py * Approx gelu * Update llama.py * Update __init__.py * Update __init__.py * Update _utils.py * Update geglu.py * Update gemma.py * Update rms_layernorm.py * Update rms_layernorm.py * Update rms_layernorm.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Update gemma.py * Fix Gemma merging * Update rms_layernorm.py * Update gemma.py * Update pyproject.toml * Layernorms * Gemma precision * Update gemma.py * sqrt * Update gemma.py * Update save.py * RoPE and Gemma precision * Update rms_layernorm.py * Fix warning * Update chat_templates.py * Update chat_templates.py * Update save.py * Update save.py * Update save.py * Update chat_templates.py * Update llama.py * model_name * Update loader.py * Tokenizer overwritten * Update llama.py * Update llama.py * Update llama.py * Update save.py * Accuracy * Revert * Update save.py * Update fast_lora.py * Update fast_lora.py * Update fast_lora.py * Update fast_lora.py * Update fast_lora.py * Update chat_templates.py * Update save.py * Update save.py * Update llama.py * Update llama.py * Account for DoRA * Update llama.py * Update save.py * GGUF incorrect * Update save.py * Update pyproject.toml * kaggle new * Update pyproject.toml * Update pyproject.toml * upcasting * Fix Colab * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml
This commit is contained in:
Daniel Han
GitHub
parent
424ffbe9d5
commit
012db22c54
6 changed files with 92 additions and 36 deletions
|
|
@ -59,7 +59,7 @@ All notebooks are **beginner friendly**! Add your dataset, click "Run All", and
|
|||
- No change of hardware. Supports NVIDIA GPUs since 2018+. Minimum CUDA Capability 7.0 (V100, T4, Titan V, RTX 20, 30, 40x, A100, H100, L40 etc) [Check your GPU!](https://developer.nvidia.com/cuda-gpus) GTX 1070, 1080 works, but is slow.
|
||||
- Works on **Linux** and **Windows** via WSL.
|
||||
- Supports 4bit and 16bit QLoRA / LoRA finetuning via [bitsandbytes](https://github.com/TimDettmers/bitsandbytes).
|
||||
- Open source trains 5x faster - see [Unsloth Pro](https://unsloth.ai/) for **30x faster training**!
|
||||
- Open source trains 5x faster - see [Unsloth Pro](https://unsloth.ai/) for up to **30x faster training**!
|
||||
- If you trained a model with 🦥Unsloth, you can use this cool sticker! <img src="https://raw.githubusercontent.com/unslothai/unsloth/main/images/made with unsloth.png" height="50" align="center" />
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -33,17 +33,17 @@ exclude = ["images*"]
|
|||
|
||||
[project.optional-dependencies]
|
||||
huggingface = [
|
||||
"tyro",
|
||||
"transformers>=4.38.2",
|
||||
"datasets>=2.16.0",
|
||||
"sentencepiece",
|
||||
"accelerate>=0.26.1",
|
||||
"trl>=0.7.9",
|
||||
"peft>=0.7.1",
|
||||
"tqdm",
|
||||
"psutil",
|
||||
"wheel>=0.42.0",
|
||||
"numpy",
|
||||
"triton",
|
||||
"accelerate>=0.26.1",
|
||||
"trl>=0.7.9",
|
||||
"peft>=0.7.1",
|
||||
]
|
||||
cu118only = [
|
||||
"xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.22.post7%2Bcu118-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9'",
|
||||
|
|
@ -128,18 +128,13 @@ cu121-torch220 = [
|
|||
kaggle = [
|
||||
"unsloth[huggingface]",
|
||||
]
|
||||
kaggle-new = [
|
||||
"unsloth[huggingface]",
|
||||
"bitsandbytes",
|
||||
]
|
||||
conda = [
|
||||
"unsloth[huggingface]",
|
||||
]
|
||||
colab = [
|
||||
"unsloth[cu121]",
|
||||
]
|
||||
colab-ampere = [
|
||||
"unsloth[cu121]",
|
||||
"packaging",
|
||||
"ninja",
|
||||
"flash-attn",
|
||||
]
|
||||
colab-torch211 = [
|
||||
"unsloth[huggingface]",
|
||||
"bitsandbytes",
|
||||
|
|
@ -166,6 +161,29 @@ colab-ampere-torch220 = [
|
|||
"ninja",
|
||||
"flash-attn",
|
||||
]
|
||||
colab-new = [
|
||||
"tyro",
|
||||
"transformers>=4.38.2",
|
||||
"datasets>=2.16.0",
|
||||
"sentencepiece",
|
||||
"tqdm",
|
||||
"psutil",
|
||||
"wheel>=0.42.0",
|
||||
"numpy",
|
||||
]
|
||||
colab-no-deps = [
|
||||
"accelerate>=0.26.1",
|
||||
"trl>=0.7.9",
|
||||
"peft>=0.7.1",
|
||||
"xformers",
|
||||
"bitsandbytes",
|
||||
]
|
||||
colab-ampere = [
|
||||
"unsloth[colab-ampere-torch220]",
|
||||
"packaging",
|
||||
"ninja",
|
||||
"flash-attn",
|
||||
]
|
||||
cu118-ampere = [
|
||||
"unsloth[huggingface]",
|
||||
"bitsandbytes",
|
||||
|
|
|
|||
|
|
@ -259,8 +259,10 @@ def get_chat_template(
|
|||
assert("Unsloth: Can only map new tokens to EOS for now. Adding new tokens is not yet supported.")
|
||||
pass
|
||||
|
||||
if tokenizer.__class__.__name__.startswith("Gemma") and chat_template == "chatml":
|
||||
chat_template = "gemma_chatml"
|
||||
IS_GEMMA = False
|
||||
if tokenizer.__class__.__name__.startswith("Gemma"):
|
||||
if chat_template == "chatml": chat_template = "gemma_chatml"
|
||||
IS_GEMMA = True
|
||||
pass
|
||||
|
||||
old_padding_side = tokenizer.padding_side
|
||||
|
|
@ -338,6 +340,12 @@ def get_chat_template(
|
|||
.replace("'user'", "'" + mapping["user"] + "'")\
|
||||
.replace("'assistant'", "'" + mapping["assistant"] + "'")
|
||||
|
||||
# Careful on Gemma
|
||||
# bos_token is a must or else losses become too high
|
||||
if IS_GEMMA and not chat_template.startswith("{{ bos_token }}"):
|
||||
chat_template = "{{ bos_token }}" + chat_template
|
||||
pass
|
||||
|
||||
_, tokenizer = patch_tokenizer(model = None, tokenizer = tokenizer)
|
||||
tokenizer.padding_side = old_padding_side
|
||||
tokenizer.chat_template = chat_template
|
||||
|
|
|
|||
|
|
@ -154,6 +154,7 @@ def GemmaModel_fast_forward_inference(
|
|||
out_weight = torch.empty_like(self.layers[0].input_layernorm.weight, dtype = torch.float32, device = "cuda")
|
||||
|
||||
hidden_states = self.embed_tokens(input_ids)
|
||||
hidden_states = hidden_states.to(self.config.torch_dtype)
|
||||
# 3072**0.5 = 55.5000 in bfloat16, whilst 55.4256 in float32
|
||||
# 2048**0.5 = 45.2500 in bfloat16, whilst 45.2548 in float32
|
||||
hidden_states *= torch.tensor(math_sqrt(self.config.hidden_size), dtype = hidden_states.dtype)
|
||||
|
|
|
|||
|
|
@ -509,7 +509,10 @@ def LlamaModel_fast_forward(
|
|||
if inputs_embeds is None:
|
||||
inputs_embeds = self.embed_tokens(input_ids)
|
||||
|
||||
# Mormalized from Gemma
|
||||
# Downcast to the correct dtype ie float32 to float16
|
||||
inputs_embeds = inputs_embeds.to(self.config.torch_dtype)
|
||||
|
||||
# Normalized from Gemma
|
||||
IS_GEMMA = self.config.model_type == "gemma"
|
||||
train_embed_tokens = self.embed_tokens.weight.requires_grad
|
||||
|
||||
|
|
@ -665,6 +668,7 @@ def LlamaModel_fast_forward_inference(
|
|||
input_ids = input_ids[:,:self.max_seq_length]
|
||||
|
||||
hidden_states = self.embed_tokens(input_ids)
|
||||
hidden_states = hidden_states.to(self.config.torch_dtype)
|
||||
|
||||
next_decoder_cache = []
|
||||
for idx, decoder_layer in enumerate(self.layers):
|
||||
|
|
@ -1334,6 +1338,7 @@ class FastLlamaModel:
|
|||
"We shall do it for you!"
|
||||
)
|
||||
train_lm_head = True
|
||||
model.model.embed_tokens.to(torch.float32, non_blocking = True)
|
||||
|
||||
elif module == "embed_tokens":
|
||||
logger.warning_once(
|
||||
|
|
@ -1341,6 +1346,7 @@ class FastLlamaModel:
|
|||
"We shall do it for you!"
|
||||
)
|
||||
train_embed_tokens = True
|
||||
model.lm_head.to(torch.float32, non_blocking = True)
|
||||
|
||||
else:
|
||||
assert(module in accepted_modules)
|
||||
|
|
@ -1477,12 +1483,12 @@ class FastLlamaModel:
|
|||
if hasattr(gate_proj, "lora_A") and \
|
||||
hasattr( up_proj, "lora_A") and \
|
||||
hasattr(down_proj, "lora_A") and \
|
||||
((gate_proj.base_layer if hasattr(gate_proj, "base_layer") else gate_proj).bias is None) and \
|
||||
(( up_proj.base_layer if hasattr( up_proj, "base_layer") else up_proj).bias is None) and \
|
||||
((down_proj.base_layer if hasattr(down_proj, "base_layer") else down_proj).bias is None) and \
|
||||
((gate_proj.lora_magnitude_vector if hasattr(gate_proj, "lora_magnitude_vector") else None) is None) and \
|
||||
(( up_proj.lora_magnitude_vector if hasattr( up_proj, "lora_magnitude_vector") else None) is None) and \
|
||||
((down_proj.lora_magnitude_vector if hasattr(down_proj, "lora_magnitude_vector") else None) is None):
|
||||
(getattr(gate_proj, "base_layer", gate_proj).bias is None) and \
|
||||
(getattr( up_proj, "base_layer", up_proj).bias is None) and \
|
||||
(getattr(down_proj, "base_layer", down_proj).bias is None) and \
|
||||
(getattr(gate_proj, "lora_magnitude_vector", None) is None) and \
|
||||
(getattr( up_proj, "lora_magnitude_vector", None) is None) and \
|
||||
(getattr(down_proj, "lora_magnitude_vector", None) is None):
|
||||
|
||||
# https://stackoverflow.com/questions/50599045/python-replacing-a-function-within-a-class-of-a-module
|
||||
layer.mlp.forward = types.MethodType(apply_lora_mlp, layer.mlp)
|
||||
|
|
@ -1501,12 +1507,12 @@ class FastLlamaModel:
|
|||
if hasattr(q_proj, "lora_A") and \
|
||||
hasattr(k_proj, "lora_A") and \
|
||||
hasattr(v_proj, "lora_A") and \
|
||||
((q_proj.base_layer if hasattr(q_proj, "base_layer") else q_proj).bias is None) and \
|
||||
((k_proj.base_layer if hasattr(k_proj, "base_layer") else k_proj).bias is None) and \
|
||||
((v_proj.base_layer if hasattr(v_proj, "base_layer") else v_proj).bias is None) and \
|
||||
((q_proj.lora_magnitude_vector if hasattr(q_proj, "lora_magnitude_vector") else None) is None) and \
|
||||
((k_proj.lora_magnitude_vector if hasattr(k_proj, "lora_magnitude_vector") else None) is None) and \
|
||||
((v_proj.lora_magnitude_vector if hasattr(v_proj, "lora_magnitude_vector") else None) is None):
|
||||
(getattr(q_proj, "base_layer", q_proj).bias is None) and \
|
||||
(getattr(q_proj, "base_layer", k_proj).bias is None) and \
|
||||
(getattr(q_proj, "base_layer", v_proj).bias is None) and \
|
||||
(getattr(q_proj, "lora_magnitude_vector", None) is None) and \
|
||||
(getattr(k_proj, "lora_magnitude_vector", None) is None) and \
|
||||
(getattr(v_proj, "lora_magnitude_vector", None) is None):
|
||||
|
||||
layer.self_attn.apply_qkv = apply_lora_qkv
|
||||
n_qkv += 1
|
||||
|
|
@ -1520,8 +1526,8 @@ class FastLlamaModel:
|
|||
# O attention patching
|
||||
o_proj = layer.self_attn.o_proj
|
||||
if hasattr(o_proj, "lora_A") and \
|
||||
((o_proj.base_layer if hasattr(o_proj, "base_layer") else o_proj).bias is None) and \
|
||||
((o_proj.lora_magnitude_vector if hasattr(o_proj, "lora_magnitude_vector") else None) is None):
|
||||
(getattr(o_proj, "base_layer", o_proj).bias is None) and \
|
||||
(getattr(o_proj, "lora_magnitude_vector", None) is None):
|
||||
|
||||
layer.self_attn.apply_o = apply_lora_o
|
||||
n_o += 1
|
||||
|
|
|
|||
|
|
@ -632,6 +632,7 @@ pass
|
|||
|
||||
|
||||
def save_to_gguf(
|
||||
model_type : str,
|
||||
model_directory : str = "unsloth_finetuned_model",
|
||||
quantization_method : str = "fast_quantized",
|
||||
first_conversion : str = "f16",
|
||||
|
|
@ -639,10 +640,18 @@ def save_to_gguf(
|
|||
):
|
||||
from transformers.models.llama.modeling_llama import logger
|
||||
|
||||
# Careful convert.py is only for Llama / Mistral based archs
|
||||
use_fast_convert = False
|
||||
if model_type == "llama": use_fast_convert = True
|
||||
elif model_type == "mistral": use_fast_convert = True
|
||||
pass
|
||||
logger.warning_once(f"Unsloth: Converting {model_type} model. Can use fast conversion = {use_fast_convert}.")
|
||||
|
||||
if quantization_method == "not_quantized": quantization_method = "f16"
|
||||
elif quantization_method == "fast_quantized": quantization_method = "q8_0"
|
||||
elif quantization_method == "quantized": quantization_method = "q4_k_m"
|
||||
elif quantization_method is None: quantization_method = "q8_0"
|
||||
pass
|
||||
|
||||
if quantization_method not in ALLOWED_QUANTS.keys():
|
||||
error = f"Unsloth: Quant method = [{quantization_method}] not supported. Choose from below:\n"
|
||||
|
|
@ -692,6 +701,12 @@ def save_to_gguf(
|
|||
pass
|
||||
pass
|
||||
|
||||
# Non llama/mistral needs can only use f32 or f16
|
||||
if not use_fast_convert and (first_conversion != "f16" or first_conversion != "f32"):
|
||||
logger.warning_once("Unsloth: We must use f16 for non Llama and Mistral models.")
|
||||
first_conversion = "f16"
|
||||
pass
|
||||
|
||||
n_cpus = psutil.cpu_count()
|
||||
if n_cpus is None: n_cpus = 1
|
||||
n_cpus *= 2
|
||||
|
|
@ -703,9 +718,15 @@ def save_to_gguf(
|
|||
f"The output location will be {final_location}\n"\
|
||||
"This will take 3 minutes...")
|
||||
|
||||
command = f"python llama.cpp/convert.py {model_directory} "\
|
||||
f"--outfile {final_location} --vocab-type hfft "\
|
||||
f"--outtype {first_conversion} --concurrency {n_cpus}"
|
||||
if use_fast_convert:
|
||||
command = f"python llama.cpp/convert.py {model_directory} "\
|
||||
f"--outfile {final_location} --vocab-type hfft "\
|
||||
f"--outtype {first_conversion} --concurrency {n_cpus}"
|
||||
else:
|
||||
command = f"python llama.cpp/convert-hf-to-gguf.py {model_directory} "\
|
||||
f"--outfile {final_location} "\
|
||||
f"--outtype {first_conversion}"
|
||||
pass
|
||||
|
||||
with subprocess.Popen(command, shell = True, stdout = subprocess.PIPE, stderr = subprocess.PIPE, bufsize = 1) as sp:
|
||||
for line in sp.stdout:
|
||||
|
|
@ -1054,7 +1075,8 @@ def unsloth_save_pretrained_gguf(
|
|||
for _ in range(3):
|
||||
gc.collect()
|
||||
|
||||
file_location = save_to_gguf(new_save_directory, quantization_method, first_conversion, makefile)
|
||||
model_type = self.config.model_type
|
||||
file_location = save_to_gguf(model_type, new_save_directory, quantization_method, first_conversion, makefile)
|
||||
|
||||
if push_to_hub:
|
||||
print("Unsloth: Uploading GGUF to Huggingface Hub...")
|
||||
|
|
@ -1154,7 +1176,8 @@ def unsloth_push_to_hub_gguf(
|
|||
for _ in range(3):
|
||||
gc.collect()
|
||||
|
||||
file_location = save_to_gguf(new_save_directory, quantization_method, first_conversion, makefile)
|
||||
model_type = self.config.model_type
|
||||
file_location = save_to_gguf(model_type, new_save_directory, quantization_method, first_conversion, makefile)
|
||||
|
||||
print("Unsloth: Uploading GGUF to Huggingface Hub...")
|
||||
username = upload_to_huggingface(
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue