Compare commits
1 commit
main
...
dh/recover
| Author | SHA1 | Date | |
|---|---|---|---|
|
a7037b6785 |
1 changed files with 136 additions and 131 deletions
273
unsloth/save.py
273
unsloth/save.py
|
|
@ -13,6 +13,7 @@
|
|||
# limitations under the License.
|
||||
|
||||
from unsloth_zoo.utils import Version
|
||||
from unsloth_zoo.utils import get_lock
|
||||
from importlib.metadata import version as importlib_version
|
||||
from unsloth_zoo.hf_utils import dtype_from_config, HAS_TORCH_DTYPE
|
||||
from unsloth_zoo.llama_cpp import (
|
||||
|
|
@ -1212,150 +1213,154 @@ def save_to_gguf(
|
|||
print(print_info)
|
||||
|
||||
# Step 1: Ensure llama.cpp is installed
|
||||
try:
|
||||
quantizer_location, converter_location = check_llama_cpp()
|
||||
print("Unsloth: llama.cpp found in the system. Skipping installation.")
|
||||
except:
|
||||
print("Unsloth: Installing llama.cpp. This might take 3 minutes...")
|
||||
if IS_KAGGLE_ENVIRONMENT:
|
||||
# Kaggle: no CUDA support due to environment limitations
|
||||
quantizer_location, converter_location = install_llama_cpp(
|
||||
gpu_support = False, print_output = print_output
|
||||
llama_cpp_lock = get_lock("llama.cpp", timeout = -1)
|
||||
with llama_cpp_lock:
|
||||
try:
|
||||
quantizer_location, converter_location = check_llama_cpp()
|
||||
print("Unsloth: llama.cpp found in the system. Skipping installation.")
|
||||
except:
|
||||
print("Unsloth: Installing llama.cpp. This might take 3 minutes...")
|
||||
if IS_KAGGLE_ENVIRONMENT:
|
||||
# Kaggle: no CUDA support due to environment limitations
|
||||
quantizer_location, converter_location = install_llama_cpp(
|
||||
gpu_support = False, print_output = print_output
|
||||
)
|
||||
else:
|
||||
quantizer_location, converter_location = install_llama_cpp(
|
||||
gpu_support = False, # GGUF conversion doesn't need CUDA
|
||||
print_output = print_output,
|
||||
)
|
||||
|
||||
# Step 2: Download and patch converter script
|
||||
save_lock = get_lock(f"save_to_gguf_{model_name}", timeout = -1)
|
||||
with save_lock:
|
||||
print("Unsloth: Preparing converter script...")
|
||||
with use_local_gguf():
|
||||
converter_path, supported_text_archs, supported_vision_archs = (
|
||||
_download_convert_hf_to_gguf()
|
||||
)
|
||||
else:
|
||||
quantizer_location, converter_location = install_llama_cpp(
|
||||
gpu_support = False, # GGUF conversion doesn't need CUDA
|
||||
|
||||
# Step 3: Initial GGUF conversion
|
||||
print(
|
||||
f"Unsloth: [1] Converting model into {first_conversion_dtype} GGUF format."
|
||||
)
|
||||
print(f"This might take 3 minutes...")
|
||||
|
||||
initial_files, is_vlm_update = convert_to_gguf(
|
||||
model_name = model_name,
|
||||
input_folder = model_directory,
|
||||
model_dtype = model_dtype,
|
||||
quantization_type = first_conversion,
|
||||
converter_location = converter_path,
|
||||
supported_text_archs = supported_text_archs,
|
||||
supported_vision_archs = supported_vision_archs,
|
||||
is_vlm = is_vlm,
|
||||
is_gpt_oss = is_gpt_oss,
|
||||
max_shard_size = "50GB",
|
||||
print_output = print_output,
|
||||
)
|
||||
|
||||
# Step 2: Download and patch converter script
|
||||
print("Unsloth: Preparing converter script...")
|
||||
with use_local_gguf():
|
||||
converter_path, supported_text_archs, supported_vision_archs = (
|
||||
_download_convert_hf_to_gguf()
|
||||
)
|
||||
|
||||
# Step 3: Initial GGUF conversion
|
||||
print(
|
||||
f"Unsloth: [1] Converting model into {first_conversion_dtype} GGUF format."
|
||||
)
|
||||
print(f"This might take 3 minutes...")
|
||||
|
||||
initial_files, is_vlm_update = convert_to_gguf(
|
||||
model_name = model_name,
|
||||
input_folder = model_directory,
|
||||
model_dtype = model_dtype,
|
||||
quantization_type = first_conversion,
|
||||
converter_location = converter_path,
|
||||
supported_text_archs = supported_text_archs,
|
||||
supported_vision_archs = supported_vision_archs,
|
||||
is_vlm = is_vlm,
|
||||
is_gpt_oss = is_gpt_oss,
|
||||
max_shard_size = "50GB",
|
||||
print_output = print_output,
|
||||
)
|
||||
# update is_vlm switch
|
||||
is_vlm = is_vlm_update
|
||||
# Check conversion success
|
||||
for file in initial_files:
|
||||
if not os.path.exists(file):
|
||||
if IS_KAGGLE_ENVIRONMENT:
|
||||
raise RuntimeError(
|
||||
f"Unsloth: Conversion failed for {file}\n"
|
||||
"You are in a Kaggle environment with limited disk space (20GB).\n"
|
||||
"Try saving to /tmp for more space or use a smaller model.\n"
|
||||
"Alternatively, save the 16bit model first, then convert manually."
|
||||
)
|
||||
else:
|
||||
raise RuntimeError(
|
||||
f"Unsloth: Conversion failed for {file}\n"
|
||||
"Please check disk space and try again."
|
||||
)
|
||||
|
||||
# Move initial GGUF files into a dedicated _gguf directory
|
||||
gguf_directory = f"{model_directory}_gguf"
|
||||
os.makedirs(gguf_directory, exist_ok = True)
|
||||
moved_files = []
|
||||
for fpath in initial_files:
|
||||
dst = os.path.join(gguf_directory, os.path.basename(fpath))
|
||||
shutil.move(fpath, dst)
|
||||
moved_files.append(dst)
|
||||
initial_files = moved_files
|
||||
|
||||
print(f"Unsloth: Initial conversion completed! Files: {initial_files}")
|
||||
|
||||
# Step 4: Additional quantizations using llama-quantize
|
||||
all_saved_locations = initial_files.copy()
|
||||
|
||||
# Get CPU count for quantization
|
||||
n_cpus = psutil.cpu_count()
|
||||
if n_cpus is None:
|
||||
n_cpus = 1
|
||||
n_cpus *= 2
|
||||
|
||||
if not is_gpt_oss:
|
||||
base_gguf = initial_files[0]
|
||||
quants_created = False
|
||||
for quant_method in quantization_method:
|
||||
if quant_method != first_conversion:
|
||||
print(
|
||||
f"Unsloth: [2] Converting GGUF {first_conversion_dtype} into {quant_method}. This might take 10 minutes..."
|
||||
)
|
||||
output_location = os.path.join(
|
||||
gguf_directory, f"{model_name}.{quant_method.upper()}.gguf"
|
||||
)
|
||||
try:
|
||||
# Use the quantize_gguf function we created
|
||||
quantized_file = quantize_gguf(
|
||||
input_gguf = base_gguf,
|
||||
output_gguf = output_location,
|
||||
quant_type = quant_method,
|
||||
quantizer_location = quantizer_location,
|
||||
print_output = print_output,
|
||||
# update is_vlm switch
|
||||
is_vlm = is_vlm_update
|
||||
# Check conversion success
|
||||
for file in initial_files:
|
||||
if not os.path.exists(file):
|
||||
if IS_KAGGLE_ENVIRONMENT:
|
||||
raise RuntimeError(
|
||||
f"Unsloth: Conversion failed for {file}\n"
|
||||
"You are in a Kaggle environment with limited disk space (20GB).\n"
|
||||
"Try saving to /tmp for more space or use a smaller model.\n"
|
||||
"Alternatively, save the 16bit model first, then convert manually."
|
||||
)
|
||||
all_saved_locations.append(quantized_file)
|
||||
quants_created = True
|
||||
except Exception as e:
|
||||
if IS_KAGGLE_ENVIRONMENT:
|
||||
raise RuntimeError(
|
||||
f"Unsloth: Quantization failed for {output_location}\n"
|
||||
"You are in a Kaggle environment, which might be the reason this is failing.\n"
|
||||
"Kaggle only provides 20GB of disk space in the working directory.\n"
|
||||
"Merging to 16bit for 7b models use 16GB of space.\n"
|
||||
"This means using `model.{save_pretrained/push_to_hub}_merged` works, but\n"
|
||||
"`model.{save_pretrained/push_to_hub}_gguf will use too much disk space.\n"
|
||||
"You can try saving it to the `/tmp` directory for larger disk space.\n"
|
||||
"I suggest you to save the 16bit model first, then use manual llama.cpp conversion.\n"
|
||||
f"Error: {e}"
|
||||
else:
|
||||
raise RuntimeError(
|
||||
f"Unsloth: Conversion failed for {file}\n"
|
||||
"Please check disk space and try again."
|
||||
)
|
||||
|
||||
# Move initial GGUF files into a dedicated _gguf directory
|
||||
gguf_directory = f"{model_directory}_gguf"
|
||||
os.makedirs(gguf_directory, exist_ok = True)
|
||||
moved_files = []
|
||||
for fpath in initial_files:
|
||||
dst = os.path.join(gguf_directory, os.path.basename(fpath))
|
||||
shutil.move(fpath, dst)
|
||||
moved_files.append(dst)
|
||||
initial_files = moved_files
|
||||
|
||||
print(f"Unsloth: Initial conversion completed! Files: {initial_files}")
|
||||
|
||||
# Step 4: Additional quantizations using llama-quantize
|
||||
all_saved_locations = initial_files.copy()
|
||||
|
||||
# Get CPU count for quantization
|
||||
n_cpus = psutil.cpu_count()
|
||||
if n_cpus is None:
|
||||
n_cpus = 1
|
||||
n_cpus *= 2
|
||||
|
||||
if not is_gpt_oss:
|
||||
base_gguf = initial_files[0]
|
||||
quants_created = False
|
||||
for quant_method in quantization_method:
|
||||
if quant_method != first_conversion:
|
||||
print(
|
||||
f"Unsloth: [2] Converting GGUF {first_conversion_dtype} into {quant_method}. This might take 10 minutes..."
|
||||
)
|
||||
output_location = os.path.join(
|
||||
gguf_directory, f"{model_name}.{quant_method.upper()}.gguf"
|
||||
)
|
||||
try:
|
||||
quantized_file = quantize_gguf(
|
||||
input_gguf = base_gguf,
|
||||
output_gguf = output_location,
|
||||
quant_type = quant_method,
|
||||
quantizer_location = quantizer_location,
|
||||
print_output = print_output,
|
||||
)
|
||||
else:
|
||||
if IS_WINDOWS:
|
||||
build_instructions = (
|
||||
f'cd "{LLAMA_CPP_DEFAULT_DIR}"\n'
|
||||
f"cmake -S . -B build -DBUILD_SHARED_LIBS=OFF\n"
|
||||
f"cmake --build build --config Release"
|
||||
all_saved_locations.append(quantized_file)
|
||||
quants_created = True
|
||||
except Exception as e:
|
||||
if IS_KAGGLE_ENVIRONMENT:
|
||||
raise RuntimeError(
|
||||
f"Unsloth: Quantization failed for {output_location}\n"
|
||||
"You are in a Kaggle environment, which might be the reason this is failing.\n"
|
||||
"Kaggle only provides 20GB of disk space in the working directory.\n"
|
||||
"Merging to 16bit for 7b models use 16GB of space.\n"
|
||||
"This means using `model.{save_pretrained/push_to_hub}_merged` works, but\n"
|
||||
"`model.{save_pretrained/push_to_hub}_gguf will use too much disk space.\n"
|
||||
"You can try saving it to the `/tmp` directory for larger disk space.\n"
|
||||
"I suggest you to save the 16bit model first, then use manual llama.cpp conversion.\n"
|
||||
f"Error: {e}"
|
||||
)
|
||||
else:
|
||||
build_instructions = f'cd "{LLAMA_CPP_DEFAULT_DIR}" && make clean && make all -j'
|
||||
if IS_WINDOWS:
|
||||
build_instructions = (
|
||||
f'cd "{LLAMA_CPP_DEFAULT_DIR}"\n'
|
||||
f"cmake -S . -B build -DBUILD_SHARED_LIBS=OFF\n"
|
||||
f"cmake --build build --config Release"
|
||||
)
|
||||
else:
|
||||
build_instructions = f'cd "{LLAMA_CPP_DEFAULT_DIR}" && make clean && make all -j'
|
||||
|
||||
raise RuntimeError(
|
||||
f"Unsloth: Quantization failed for {output_location}\n"
|
||||
"You might have to compile llama.cpp yourself, then run this again.\n"
|
||||
"You do not need to close this Python program. Run the following commands in a new terminal:\n"
|
||||
f'git clone --recursive https://github.com/ggerganov/llama.cpp "{LLAMA_CPP_DEFAULT_DIR}"\n'
|
||||
f"{build_instructions}\n"
|
||||
"Once that's done, redo the quantization.\n"
|
||||
f"Error: {e}"
|
||||
)
|
||||
print("Unsloth: Model files cleanup...")
|
||||
if quants_created:
|
||||
all_saved_locations.remove(base_gguf)
|
||||
Path(base_gguf).unlink(missing_ok = True)
|
||||
raise RuntimeError(
|
||||
f"Unsloth: Quantization failed for {output_location}\n"
|
||||
"You might have to compile llama.cpp yourself, then run this again.\n"
|
||||
"You do not need to close this Python program. Run the following commands in a new terminal:\n"
|
||||
f'git clone --recursive https://github.com/ggerganov/llama.cpp "{LLAMA_CPP_DEFAULT_DIR}"\n'
|
||||
f"{build_instructions}\n"
|
||||
"Once that's done, redo the quantization.\n"
|
||||
f"Error: {e}"
|
||||
)
|
||||
print("Unsloth: Model files cleanup...")
|
||||
if quants_created:
|
||||
all_saved_locations.remove(base_gguf)
|
||||
Path(base_gguf).unlink(missing_ok = True)
|
||||
|
||||
# flip the list to get [text_model, mmproj] order. for text models stays the same.
|
||||
all_saved_locations.reverse()
|
||||
else:
|
||||
print("Unsloth: GPT-OSS model - skipping additional quantizations")
|
||||
# flip the list to get [text_model, mmproj] order. for text models stays the same.
|
||||
all_saved_locations.reverse()
|
||||
else:
|
||||
print("Unsloth: GPT-OSS model - skipping additional quantizations")
|
||||
|
||||
if is_gpt_oss:
|
||||
want_full_precision = True
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue