Compare commits
6 commits
main
...
studio/bat
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
aae1059d90 | ||
|
|
2cb2fa8a76 | ||
|
|
92f22f66c0 | ||
|
|
b9e5a13d4e | ||
|
|
176ae1f17a | ||
|
|
9a818e7155 |
8 changed files with 256 additions and 75 deletions
|
|
@ -12,6 +12,7 @@ import structlog
|
|||
from loggers import get_logger
|
||||
import os
|
||||
import shutil
|
||||
import traceback
|
||||
from pathlib import Path
|
||||
from typing import Optional, Tuple, List
|
||||
from peft import PeftModel, PeftModelForCausalLM
|
||||
|
|
@ -486,17 +487,28 @@ class ExportBackend:
|
|||
def export_gguf(
|
||||
self,
|
||||
save_directory: str,
|
||||
quantization_method: str = "Q4_K_M",
|
||||
quantization_method: List[str],
|
||||
push_to_hub: bool = False,
|
||||
repo_id: Optional[str] = None,
|
||||
hf_token: Optional[str] = None,
|
||||
) -> Tuple[bool, str]:
|
||||
"""
|
||||
Export model in GGUF format.
|
||||
Export model in GGUF format as a single batch call.
|
||||
|
||||
Accepts a list of lowercase quantization methods (already
|
||||
normalized by `models.export.normalize_gguf_quantization_method`
|
||||
at the route boundary). `save_pretrained_gguf` is called once
|
||||
with the full list, producing all requested formats from a
|
||||
single weight merge.
|
||||
|
||||
On any `save_pretrained_gguf` exception the error is captured
|
||||
but the relocation block still runs in a `finally` so any
|
||||
partial outputs already written to the repo-root cwd are moved
|
||||
into the user's export directory rather than stranded.
|
||||
|
||||
Args:
|
||||
save_directory: Local directory to save model
|
||||
quantization_method: GGUF quantization method (e.g., "Q4_K_M")
|
||||
quantization_method: List of lowercase GGUF quantization methods
|
||||
push_to_hub: Whether to push to Hugging Face Hub
|
||||
repo_id: Hub repository ID
|
||||
hf_token: Hugging Face token
|
||||
|
|
@ -507,53 +519,65 @@ class ExportBackend:
|
|||
if not self.current_model or not self.current_tokenizer:
|
||||
return False, "No model loaded. Please select a checkpoint first."
|
||||
|
||||
# Defensive lowercase: route already does this, but direct
|
||||
# callers (worker default, internal tests) may pass raw casing.
|
||||
quant_methods = [q.lower() for q in quantization_method]
|
||||
|
||||
if not save_directory:
|
||||
return False, "save_directory is required for GGUF export"
|
||||
|
||||
try:
|
||||
# Convert quantization method to lowercase for unsloth
|
||||
quant_method = quantization_method.lower()
|
||||
save_directory = str(resolve_export_dir(save_directory))
|
||||
# Resolve to absolute path so unsloth's relative-path internals
|
||||
# (check_llama_cpp, use_local_gguf, _download_convert_hf_to_gguf)
|
||||
# all resolve against the repo root cwd, NOT the export directory.
|
||||
abs_save_dir = os.path.abspath(save_directory)
|
||||
logger.info(f"Saving GGUF model locally to: {abs_save_dir}")
|
||||
|
||||
# Save locally if requested
|
||||
if save_directory:
|
||||
save_directory = str(resolve_export_dir(save_directory))
|
||||
# Resolve to absolute path so unsloth's relative-path internals
|
||||
# (check_llama_cpp, use_local_gguf, _download_convert_hf_to_gguf)
|
||||
# all resolve against the repo root cwd, NOT the export directory.
|
||||
abs_save_dir = os.path.abspath(save_directory)
|
||||
logger.info(f"Saving GGUF model locally to: {abs_save_dir}")
|
||||
# Create the directory if it doesn't exist
|
||||
ensure_dir(Path(abs_save_dir))
|
||||
|
||||
# Create the directory if it doesn't exist
|
||||
ensure_dir(Path(abs_save_dir))
|
||||
# On WSL, patch out sudo check before llama.cpp build
|
||||
_apply_wsl_sudo_patch()
|
||||
|
||||
# On WSL, patch out sudo check before llama.cpp build
|
||||
_apply_wsl_sudo_patch()
|
||||
# Snapshot existing .gguf files in cwd before conversion.
|
||||
# unsloth's convert_to_gguf writes output files relative to
|
||||
# cwd (repo root), so we diff afterwards and relocate them.
|
||||
cwd = os.getcwd()
|
||||
pre_existing_ggufs = set(glob.glob(os.path.join(cwd, "*.gguf")))
|
||||
|
||||
# Snapshot existing .gguf files in cwd before conversion.
|
||||
# unsloth's convert_to_gguf writes output files relative to
|
||||
# cwd (repo root), so we diff afterwards and relocate them.
|
||||
cwd = os.getcwd()
|
||||
pre_existing_ggufs = set(glob.glob(os.path.join(cwd, "*.gguf")))
|
||||
# Pass absolute path — no os.chdir needed.
|
||||
# unsloth saves intermediate HF model files into model_save_path.
|
||||
model_save_path = os.path.join(abs_save_dir, "model")
|
||||
|
||||
# Pass absolute path — no os.chdir needed.
|
||||
# unsloth saves intermediate HF model files into model_save_path.
|
||||
# unsloth-zoo's check_llama_cpp() uses ~/.unsloth/llama.cpp by default.
|
||||
model_save_path = os.path.join(abs_save_dir, "model")
|
||||
save_exception: Optional[BaseException] = None
|
||||
try:
|
||||
self.current_model.save_pretrained_gguf(
|
||||
model_save_path,
|
||||
self.current_tokenizer,
|
||||
quantization_method = quant_method,
|
||||
quantization_method = quant_methods,
|
||||
)
|
||||
|
||||
except BaseException as save_exc:
|
||||
save_exception = save_exc
|
||||
logger.error(f"GGUF batch export failed: {save_exc}")
|
||||
logger.error(traceback.format_exc())
|
||||
finally:
|
||||
# Relocate GGUF artifacts into the export directory.
|
||||
# convert_to_gguf writes .gguf files to cwd (repo root)
|
||||
# because --outfile is a relative path like "model.Q4_K_M.gguf".
|
||||
# Runs on BOTH success and failure so partial outputs
|
||||
# produced before a batch failure are visible to the
|
||||
# user in the export dir instead of stranded in cwd.
|
||||
new_ggufs = (
|
||||
set(glob.glob(os.path.join(cwd, "*.gguf"))) - pre_existing_ggufs
|
||||
)
|
||||
for src in sorted(new_ggufs):
|
||||
dest = os.path.join(abs_save_dir, os.path.basename(src))
|
||||
shutil.move(src, dest)
|
||||
logger.info(
|
||||
f"Relocated GGUF: {os.path.basename(src)} → {abs_save_dir}/"
|
||||
)
|
||||
try:
|
||||
shutil.move(src, dest)
|
||||
logger.info(
|
||||
f"Relocated GGUF: {os.path.basename(src)} → {abs_save_dir}/"
|
||||
)
|
||||
except OSError as move_exc:
|
||||
logger.warning(f"Failed to relocate {src}: {move_exc}")
|
||||
|
||||
# Flatten any .gguf files from subdirectories into abs_save_dir.
|
||||
# save_pretrained_gguf may create subdirs (e.g. model_gguf/)
|
||||
|
|
@ -563,8 +587,11 @@ class ExportBackend:
|
|||
continue
|
||||
for src in sub.glob("*.gguf"):
|
||||
dest = os.path.join(abs_save_dir, src.name)
|
||||
shutil.move(str(src), dest)
|
||||
logger.info(f"Relocated GGUF: {src.name} → {abs_save_dir}/")
|
||||
try:
|
||||
shutil.move(str(src), dest)
|
||||
logger.info(f"Relocated GGUF: {src.name} → {abs_save_dir}/")
|
||||
except OSError as move_exc:
|
||||
logger.warning(f"Failed to relocate {src}: {move_exc}")
|
||||
# Clean up the subdirectory (intermediate HF files, etc.)
|
||||
shutil.rmtree(str(sub), ignore_errors = True)
|
||||
logger.info(f"Cleaned up subdirectory: {sub.name}")
|
||||
|
|
@ -584,25 +611,31 @@ class ExportBackend:
|
|||
modelfile = gguf_dir / "Modelfile"
|
||||
if modelfile.is_file():
|
||||
shutil.move(
|
||||
str(modelfile), os.path.join(abs_save_dir, "Modelfile")
|
||||
str(modelfile),
|
||||
os.path.join(abs_save_dir, "Modelfile"),
|
||||
)
|
||||
logger.info(f"Relocated Modelfile → {abs_save_dir}/")
|
||||
shutil.rmtree(str(gguf_dir), ignore_errors = True)
|
||||
logger.info(f"Cleaned up intermediate GGUF dir: {gguf_dir}")
|
||||
|
||||
# Write export metadata so the Chat page can identify the base model
|
||||
self._write_export_metadata(abs_save_dir)
|
||||
# After the finally block: if save raised, return failure now.
|
||||
# Partial outputs (if any) have already been relocated above.
|
||||
if save_exception is not None:
|
||||
return False, f"GGUF export failed: {save_exception}"
|
||||
|
||||
# Log final file locations (after relocation) so it's clear
|
||||
# where the GGUF files actually ended up.
|
||||
final_ggufs = sorted(glob.glob(os.path.join(abs_save_dir, "*.gguf")))
|
||||
logger.info(
|
||||
"GGUF export complete. Final files in %s:\n %s",
|
||||
abs_save_dir,
|
||||
"\n ".join(os.path.basename(f) for f in final_ggufs) or "(none)",
|
||||
)
|
||||
# Write export metadata so the Chat page can identify the base model
|
||||
self._write_export_metadata(abs_save_dir)
|
||||
|
||||
# Push to hub if requested
|
||||
# Log final file locations (after relocation) so it's clear
|
||||
# where the GGUF files actually ended up.
|
||||
final_ggufs = sorted(glob.glob(os.path.join(abs_save_dir, "*.gguf")))
|
||||
logger.info(
|
||||
"GGUF export complete. Final files in %s:\n %s",
|
||||
abs_save_dir,
|
||||
"\n ".join(os.path.basename(f) for f in final_ggufs) or "(none)",
|
||||
)
|
||||
|
||||
# Push to hub if requested (single batch call)
|
||||
if push_to_hub:
|
||||
if not repo_id or not hf_token:
|
||||
return (
|
||||
|
|
@ -615,17 +648,18 @@ class ExportBackend:
|
|||
self.current_model.push_to_hub_gguf(
|
||||
repo_id,
|
||||
self.current_tokenizer,
|
||||
quantization_method = quant_method,
|
||||
quantization_method = quant_methods,
|
||||
token = hf_token,
|
||||
)
|
||||
logger.info(f"GGUF model pushed successfully to {repo_id}")
|
||||
|
||||
return True, f"GGUF model exported successfully ({quantization_method})"
|
||||
return (
|
||||
True,
|
||||
f"GGUF model exported successfully ({', '.join(quant_methods)})",
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error exporting GGUF model: {e}")
|
||||
import traceback
|
||||
|
||||
logger.error(traceback.format_exc())
|
||||
return False, f"GGUF export failed: {str(e)}"
|
||||
|
||||
|
|
|
|||
|
|
@ -310,12 +310,16 @@ class ExportOrchestrator:
|
|||
def export_gguf(
|
||||
self,
|
||||
save_directory: str,
|
||||
quantization_method: str = "Q4_K_M",
|
||||
quantization_method: List[str],
|
||||
push_to_hub: bool = False,
|
||||
repo_id: Optional[str] = None,
|
||||
hf_token: Optional[str] = None,
|
||||
) -> Tuple[bool, str]:
|
||||
"""Export model in GGUF format."""
|
||||
"""
|
||||
Export model in GGUF format. The caller must supply a normalized
|
||||
list of lowercase quantization method strings (see
|
||||
`models.export.normalize_gguf_quantization_method`).
|
||||
"""
|
||||
return self._run_export(
|
||||
"gguf",
|
||||
{
|
||||
|
|
|
|||
|
|
@ -139,7 +139,7 @@ def _handle_export(backend, cmd: dict, resp_queue: Any) -> None:
|
|||
elif export_type == "gguf":
|
||||
success, message = backend.export_gguf(
|
||||
save_directory = cmd.get("save_directory", ""),
|
||||
quantization_method = cmd.get("quantization_method", "Q4_K_M"),
|
||||
quantization_method = cmd.get("quantization_method", ["Q4_K_M"]),
|
||||
push_to_hub = cmd.get("push_to_hub", False),
|
||||
repo_id = cmd.get("repo_id"),
|
||||
hf_token = cmd.get("hf_token"),
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ Pydantic schemas for Export API.
|
|||
"""
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
from typing import List, Optional, Literal, Dict, Any
|
||||
from typing import Any, Dict, List, Literal, Optional, Set, Union
|
||||
|
||||
|
||||
class LoadCheckpointRequest(BaseModel):
|
||||
|
|
@ -108,9 +108,12 @@ class ExportGGUFRequest(BaseModel):
|
|||
...,
|
||||
description = "Directory where GGUF files will be saved",
|
||||
)
|
||||
quantization_method: str = Field(
|
||||
quantization_method: Union[str, List[str]] = Field(
|
||||
"Q4_K_M",
|
||||
description = 'GGUF quantization method (e.g. "Q4_K_M")',
|
||||
description = (
|
||||
"GGUF quantization method, or list of methods to produce in a "
|
||||
'single batch (e.g. "Q4_K_M" or ["Q4_K_M", "BF16"]).'
|
||||
),
|
||||
)
|
||||
push_to_hub: bool = Field(
|
||||
False,
|
||||
|
|
@ -130,3 +133,39 @@ class ExportLoRAAdapterRequest(ExportCommonOptions):
|
|||
"""Request for exporting only the LoRA adapter (not merged)."""
|
||||
|
||||
# Uses fields from ExportCommonOptions only
|
||||
|
||||
|
||||
def normalize_gguf_quantization_method(
|
||||
value: Union[str, List[str]],
|
||||
) -> List[str]:
|
||||
"""
|
||||
Normalize a GGUF `quantization_method` value to a lowercase, deduplicated
|
||||
list suitable for `save_pretrained_gguf`.
|
||||
|
||||
Accepts either a single string (legacy single-format callers) or a list
|
||||
of strings (batch callers). Returns a list preserving first-seen order.
|
||||
|
||||
Raises:
|
||||
ValueError: if the resulting list is empty. The route handler catches
|
||||
this and returns `HTTPException(status_code=400, ...)` so clients
|
||||
get a clear 400 instead of a generic 500 from deeper in the stack.
|
||||
"""
|
||||
if isinstance(value, str):
|
||||
methods = [value]
|
||||
else:
|
||||
methods = list(value)
|
||||
|
||||
seen: Set[str] = set()
|
||||
normalized: List[str] = []
|
||||
for method in methods:
|
||||
lowered = method.lower()
|
||||
if lowered not in seen:
|
||||
seen.add(lowered)
|
||||
normalized.append(lowered)
|
||||
|
||||
if not normalized:
|
||||
raise ValueError(
|
||||
"quantization_method must contain at least one format",
|
||||
)
|
||||
|
||||
return normalized
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@ from models import (
|
|||
ExportGGUFRequest,
|
||||
ExportLoRAAdapterRequest,
|
||||
)
|
||||
from models.export import normalize_gguf_quantization_method
|
||||
|
||||
router = APIRouter()
|
||||
logger = get_logger(__name__)
|
||||
|
|
@ -253,11 +254,18 @@ async def export_gguf(
|
|||
|
||||
Wraps ExportBackend.export_gguf.
|
||||
"""
|
||||
try:
|
||||
normalized_methods = normalize_gguf_quantization_method(
|
||||
request.quantization_method,
|
||||
)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code = 400, detail = str(exc))
|
||||
|
||||
try:
|
||||
backend = get_export_backend()
|
||||
success, message = backend.export_gguf(
|
||||
save_directory = request.save_directory,
|
||||
quantization_method = request.quantization_method,
|
||||
quantization_method = normalized_methods,
|
||||
push_to_hub = request.push_to_hub,
|
||||
repo_id = request.repo_id,
|
||||
hf_token = request.hf_token,
|
||||
|
|
|
|||
98
studio/backend/tests/test_export_gguf_batching.py
Normal file
98
studio/backend/tests/test_export_gguf_batching.py
Normal file
|
|
@ -0,0 +1,98 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
"""
|
||||
Unit tests for the GGUF export quantization_method normalization helper.
|
||||
|
||||
Tests the pure helper function `normalize_gguf_quantization_method` in
|
||||
isolation — no FastAPI app, no TestClient, no route invocation, no stubs.
|
||||
The helper owns the Union[str, List[str]] coercion, case normalization,
|
||||
deduplication, and empty-list rejection that `routes/export.py` relies on.
|
||||
|
||||
Corresponds to the design spec at
|
||||
`.claude/specs/2026-04-14-studio-batch-gguf-exports-design.md`.
|
||||
|
||||
No GPU, no network, no heavy imports — runs in milliseconds.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
# Match conftest.py's sys.path handling so flat imports from the backend
|
||||
# root (e.g. `from models.export import ...`) resolve.
|
||||
_backend_root = Path(__file__).resolve().parent.parent
|
||||
if str(_backend_root) not in sys.path:
|
||||
sys.path.insert(0, str(_backend_root))
|
||||
|
||||
from models.export import (
|
||||
ExportGGUFRequest,
|
||||
normalize_gguf_quantization_method,
|
||||
)
|
||||
|
||||
|
||||
class TestNormalizeGGUFQuantizationMethod:
|
||||
"""Covers the five normalization contracts the route handler relies on."""
|
||||
|
||||
def test_string_input_wraps_and_lowercases(self):
|
||||
"""A single string becomes a single-element lowercase list."""
|
||||
assert normalize_gguf_quantization_method("Q4_K_M") == ["q4_k_m"]
|
||||
|
||||
def test_list_input_lowercases_each_element(self):
|
||||
"""A list of strings is lowercased element-wise, order preserved."""
|
||||
assert normalize_gguf_quantization_method(["Q4_K_M", "BF16"]) == [
|
||||
"q4_k_m",
|
||||
"bf16",
|
||||
]
|
||||
|
||||
def test_duplicate_formats_deduped_preserving_first_seen_order(self):
|
||||
"""Dedup runs after lowercasing; first occurrence wins the position."""
|
||||
result = normalize_gguf_quantization_method(
|
||||
["Q4_K_M", "q4_k_m", "Q8_0", "Q4_K_M"],
|
||||
)
|
||||
assert result == ["q4_k_m", "q8_0"]
|
||||
|
||||
def test_empty_list_raises_valueerror(self):
|
||||
"""Empty input is rejected so the route can map it to HTTP 400."""
|
||||
with pytest.raises(ValueError, match = "at least one"):
|
||||
normalize_gguf_quantization_method([])
|
||||
|
||||
def test_single_element_list_passes_through(self):
|
||||
"""Single-element list is a valid input (equivalent to the string form)."""
|
||||
assert normalize_gguf_quantization_method(["BF16"]) == ["bf16"]
|
||||
|
||||
|
||||
class TestExportGGUFRequestDefault:
|
||||
"""Integration between Pydantic default and the normalization helper."""
|
||||
|
||||
def test_pydantic_default_normalizes_to_lowercase_list(self):
|
||||
"""When the caller omits quantization_method, the default string flows
|
||||
through the helper and becomes a one-element lowercase list."""
|
||||
req = ExportGGUFRequest(save_directory = "/tmp/x")
|
||||
assert normalize_gguf_quantization_method(req.quantization_method) == [
|
||||
"q4_k_m",
|
||||
]
|
||||
|
||||
def test_pydantic_accepts_string_input(self):
|
||||
"""Legacy scripted callers can still send a single-format string."""
|
||||
req = ExportGGUFRequest(
|
||||
save_directory = "/tmp/x",
|
||||
quantization_method = "Q4_K_M",
|
||||
)
|
||||
assert normalize_gguf_quantization_method(req.quantization_method) == [
|
||||
"q4_k_m",
|
||||
]
|
||||
|
||||
def test_pydantic_accepts_list_input(self):
|
||||
"""New callers send the list form."""
|
||||
req = ExportGGUFRequest(
|
||||
save_directory = "/tmp/x",
|
||||
quantization_method = ["Q4_K_M", "BF16"],
|
||||
)
|
||||
assert normalize_gguf_quantization_method(req.quantization_method) == [
|
||||
"q4_k_m",
|
||||
"bf16",
|
||||
]
|
||||
|
|
@ -99,7 +99,7 @@ export async function exportBase(params: {
|
|||
|
||||
export async function exportGGUF(params: {
|
||||
save_directory: string;
|
||||
quantization_method: string;
|
||||
quantization_method: string[];
|
||||
push_to_hub?: boolean;
|
||||
repo_id?: string | null;
|
||||
hf_token?: string | null;
|
||||
|
|
|
|||
|
|
@ -454,15 +454,13 @@ export function ExportPage() {
|
|||
});
|
||||
}
|
||||
} else if (exportMethod === "gguf") {
|
||||
for (const quant of quantLevels) {
|
||||
await exportGGUF({
|
||||
save_directory: saveDir,
|
||||
quantization_method: quant,
|
||||
push_to_hub: pushToHub,
|
||||
repo_id: repoId,
|
||||
hf_token: token,
|
||||
});
|
||||
}
|
||||
await exportGGUF({
|
||||
save_directory: saveDir,
|
||||
quantization_method: quantLevels,
|
||||
push_to_hub: pushToHub,
|
||||
repo_id: repoId,
|
||||
hf_token: token,
|
||||
});
|
||||
} else if (exportMethod === "lora") {
|
||||
await exportLoRA({
|
||||
save_directory: saveDir,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue