Compare commits

...
Sign in to create a new pull request.

6 commits

Author SHA1 Message Date
Roland Tannous
aae1059d90 studio(frontend): send all GGUF formats in one request
Drop the per-format serial loop that called exportGGUF once per item
in quantLevels. quantLevels is already a string[]; pass it directly so
multi-format selections become one POST instead of N sequential POSTs.

Eliminates the wasted weight-merge between formats and lets the backend
route all formats through save_pretrained_gguf's native list handling.
Closes the studio-side half of the batch GGUF export work.
2026-04-14 23:39:13 +04:00
Roland Tannous
2cb2fa8a76 studio(frontend): exportGGUF now takes quantization_method: string[]
Widen the API client signature to match the backend's new list-accepting
contract. The serial-loop caller in export-page.tsx is broken by this
commit on purpose — the next commit removes the loop and passes the
full quantLevels array in one call.
2026-04-14 23:34:09 +04:00
Roland Tannous
92f22f66c0 studio: ExportBackend.export_gguf batch call + try/finally relocation
Accept List[str] and invoke save_pretrained_gguf exactly once with the
full list. Wrap the save call in a nested try/finally so the GGUF
relocation block (cwd diff + subdir flatten + _gguf dir cleanup) runs
even when the batch call raises, ensuring partial outputs are moved
into the user's export directory instead of stranded in the repo root.

push_to_hub_gguf is also called once with the full list — unsloth
handles the list internally via save_pretrained_gguf.

Success message joins the list for readable multi-format output.

Completes the backend side of the studio GGUF batching change; frontend
still loops serial calls until the next commits.
2026-04-14 23:12:59 +04:00
Roland Tannous
b9e5a13d4e studio: thread List[str] quantization_method through orchestrator + worker
ExportOrchestrator.export_gguf's parameter type is widened from str to
List[str]. Worker's cmd.get default becomes ["Q4_K_M"] to match.

The orchestrator pickles the list into the mp.Queue cmd dict; the
subprocess worker hands it to ExportBackend.export_gguf unchanged.
Both layers are pure passthrough for this field.
2026-04-14 22:46:29 +04:00
Roland Tannous
176ae1f17a studio: route /export/gguf through quantization_method normalizer
Call normalize_gguf_quantization_method on request.quantization_method
before invoking the export backend. Empty lists surface as HTTP 400
via the helper's ValueError. Legacy single-string callers are wrapped
transparently; batch callers pass the list directly.

After this commit the route always hands the backend a List[str].
2026-04-14 22:39:02 +04:00
Roland Tannous
9a818e7155 studio: add normalize_gguf_quantization_method helper + tests
Introduce a pure helper that coerces Union[str, List[str]] inputs to
a lowercase, deduplicated list, rejecting empty lists via ValueError.
The ExportGGUFRequest.quantization_method field type is widened to
Union[str, List[str]] to accept batch inputs without breaking legacy
single-format callers.

The helper is the single point of normalization for the GGUF export
API boundary; routes/export.py will call it in the next commit.

Hermetic pytest at studio/backend/tests/test_export_gguf_batching.py
covers string-to-list wrapping, list lowercasing, order-preserving
dedup, empty-list rejection, and the three Pydantic-integration paths
(default, string input, list input).
2026-04-14 22:31:20 +04:00
8 changed files with 256 additions and 75 deletions

View file

@ -12,6 +12,7 @@ import structlog
from loggers import get_logger
import os
import shutil
import traceback
from pathlib import Path
from typing import Optional, Tuple, List
from peft import PeftModel, PeftModelForCausalLM
@ -486,17 +487,28 @@ class ExportBackend:
def export_gguf(
self,
save_directory: str,
quantization_method: str = "Q4_K_M",
quantization_method: List[str],
push_to_hub: bool = False,
repo_id: Optional[str] = None,
hf_token: Optional[str] = None,
) -> Tuple[bool, str]:
"""
Export model in GGUF format.
Export model in GGUF format as a single batch call.
Accepts a list of lowercase quantization methods (already
normalized by `models.export.normalize_gguf_quantization_method`
at the route boundary). `save_pretrained_gguf` is called once
with the full list, producing all requested formats from a
single weight merge.
On any `save_pretrained_gguf` exception the error is captured
but the relocation block still runs in a `finally` so any
partial outputs already written to the repo-root cwd are moved
into the user's export directory rather than stranded.
Args:
save_directory: Local directory to save model
quantization_method: GGUF quantization method (e.g., "Q4_K_M")
quantization_method: List of lowercase GGUF quantization methods
push_to_hub: Whether to push to Hugging Face Hub
repo_id: Hub repository ID
hf_token: Hugging Face token
@ -507,53 +519,65 @@ class ExportBackend:
if not self.current_model or not self.current_tokenizer:
return False, "No model loaded. Please select a checkpoint first."
# Defensive lowercase: route already does this, but direct
# callers (worker default, internal tests) may pass raw casing.
quant_methods = [q.lower() for q in quantization_method]
if not save_directory:
return False, "save_directory is required for GGUF export"
try:
# Convert quantization method to lowercase for unsloth
quant_method = quantization_method.lower()
save_directory = str(resolve_export_dir(save_directory))
# Resolve to absolute path so unsloth's relative-path internals
# (check_llama_cpp, use_local_gguf, _download_convert_hf_to_gguf)
# all resolve against the repo root cwd, NOT the export directory.
abs_save_dir = os.path.abspath(save_directory)
logger.info(f"Saving GGUF model locally to: {abs_save_dir}")
# Save locally if requested
if save_directory:
save_directory = str(resolve_export_dir(save_directory))
# Resolve to absolute path so unsloth's relative-path internals
# (check_llama_cpp, use_local_gguf, _download_convert_hf_to_gguf)
# all resolve against the repo root cwd, NOT the export directory.
abs_save_dir = os.path.abspath(save_directory)
logger.info(f"Saving GGUF model locally to: {abs_save_dir}")
# Create the directory if it doesn't exist
ensure_dir(Path(abs_save_dir))
# Create the directory if it doesn't exist
ensure_dir(Path(abs_save_dir))
# On WSL, patch out sudo check before llama.cpp build
_apply_wsl_sudo_patch()
# On WSL, patch out sudo check before llama.cpp build
_apply_wsl_sudo_patch()
# Snapshot existing .gguf files in cwd before conversion.
# unsloth's convert_to_gguf writes output files relative to
# cwd (repo root), so we diff afterwards and relocate them.
cwd = os.getcwd()
pre_existing_ggufs = set(glob.glob(os.path.join(cwd, "*.gguf")))
# Snapshot existing .gguf files in cwd before conversion.
# unsloth's convert_to_gguf writes output files relative to
# cwd (repo root), so we diff afterwards and relocate them.
cwd = os.getcwd()
pre_existing_ggufs = set(glob.glob(os.path.join(cwd, "*.gguf")))
# Pass absolute path — no os.chdir needed.
# unsloth saves intermediate HF model files into model_save_path.
model_save_path = os.path.join(abs_save_dir, "model")
# Pass absolute path — no os.chdir needed.
# unsloth saves intermediate HF model files into model_save_path.
# unsloth-zoo's check_llama_cpp() uses ~/.unsloth/llama.cpp by default.
model_save_path = os.path.join(abs_save_dir, "model")
save_exception: Optional[BaseException] = None
try:
self.current_model.save_pretrained_gguf(
model_save_path,
self.current_tokenizer,
quantization_method = quant_method,
quantization_method = quant_methods,
)
except BaseException as save_exc:
save_exception = save_exc
logger.error(f"GGUF batch export failed: {save_exc}")
logger.error(traceback.format_exc())
finally:
# Relocate GGUF artifacts into the export directory.
# convert_to_gguf writes .gguf files to cwd (repo root)
# because --outfile is a relative path like "model.Q4_K_M.gguf".
# Runs on BOTH success and failure so partial outputs
# produced before a batch failure are visible to the
# user in the export dir instead of stranded in cwd.
new_ggufs = (
set(glob.glob(os.path.join(cwd, "*.gguf"))) - pre_existing_ggufs
)
for src in sorted(new_ggufs):
dest = os.path.join(abs_save_dir, os.path.basename(src))
shutil.move(src, dest)
logger.info(
f"Relocated GGUF: {os.path.basename(src)}{abs_save_dir}/"
)
try:
shutil.move(src, dest)
logger.info(
f"Relocated GGUF: {os.path.basename(src)}{abs_save_dir}/"
)
except OSError as move_exc:
logger.warning(f"Failed to relocate {src}: {move_exc}")
# Flatten any .gguf files from subdirectories into abs_save_dir.
# save_pretrained_gguf may create subdirs (e.g. model_gguf/)
@ -563,8 +587,11 @@ class ExportBackend:
continue
for src in sub.glob("*.gguf"):
dest = os.path.join(abs_save_dir, src.name)
shutil.move(str(src), dest)
logger.info(f"Relocated GGUF: {src.name}{abs_save_dir}/")
try:
shutil.move(str(src), dest)
logger.info(f"Relocated GGUF: {src.name}{abs_save_dir}/")
except OSError as move_exc:
logger.warning(f"Failed to relocate {src}: {move_exc}")
# Clean up the subdirectory (intermediate HF files, etc.)
shutil.rmtree(str(sub), ignore_errors = True)
logger.info(f"Cleaned up subdirectory: {sub.name}")
@ -584,25 +611,31 @@ class ExportBackend:
modelfile = gguf_dir / "Modelfile"
if modelfile.is_file():
shutil.move(
str(modelfile), os.path.join(abs_save_dir, "Modelfile")
str(modelfile),
os.path.join(abs_save_dir, "Modelfile"),
)
logger.info(f"Relocated Modelfile → {abs_save_dir}/")
shutil.rmtree(str(gguf_dir), ignore_errors = True)
logger.info(f"Cleaned up intermediate GGUF dir: {gguf_dir}")
# Write export metadata so the Chat page can identify the base model
self._write_export_metadata(abs_save_dir)
# After the finally block: if save raised, return failure now.
# Partial outputs (if any) have already been relocated above.
if save_exception is not None:
return False, f"GGUF export failed: {save_exception}"
# Log final file locations (after relocation) so it's clear
# where the GGUF files actually ended up.
final_ggufs = sorted(glob.glob(os.path.join(abs_save_dir, "*.gguf")))
logger.info(
"GGUF export complete. Final files in %s:\n %s",
abs_save_dir,
"\n ".join(os.path.basename(f) for f in final_ggufs) or "(none)",
)
# Write export metadata so the Chat page can identify the base model
self._write_export_metadata(abs_save_dir)
# Push to hub if requested
# Log final file locations (after relocation) so it's clear
# where the GGUF files actually ended up.
final_ggufs = sorted(glob.glob(os.path.join(abs_save_dir, "*.gguf")))
logger.info(
"GGUF export complete. Final files in %s:\n %s",
abs_save_dir,
"\n ".join(os.path.basename(f) for f in final_ggufs) or "(none)",
)
# Push to hub if requested (single batch call)
if push_to_hub:
if not repo_id or not hf_token:
return (
@ -615,17 +648,18 @@ class ExportBackend:
self.current_model.push_to_hub_gguf(
repo_id,
self.current_tokenizer,
quantization_method = quant_method,
quantization_method = quant_methods,
token = hf_token,
)
logger.info(f"GGUF model pushed successfully to {repo_id}")
return True, f"GGUF model exported successfully ({quantization_method})"
return (
True,
f"GGUF model exported successfully ({', '.join(quant_methods)})",
)
except Exception as e:
logger.error(f"Error exporting GGUF model: {e}")
import traceback
logger.error(traceback.format_exc())
return False, f"GGUF export failed: {str(e)}"

View file

@ -310,12 +310,16 @@ class ExportOrchestrator:
def export_gguf(
self,
save_directory: str,
quantization_method: str = "Q4_K_M",
quantization_method: List[str],
push_to_hub: bool = False,
repo_id: Optional[str] = None,
hf_token: Optional[str] = None,
) -> Tuple[bool, str]:
"""Export model in GGUF format."""
"""
Export model in GGUF format. The caller must supply a normalized
list of lowercase quantization method strings (see
`models.export.normalize_gguf_quantization_method`).
"""
return self._run_export(
"gguf",
{

View file

@ -139,7 +139,7 @@ def _handle_export(backend, cmd: dict, resp_queue: Any) -> None:
elif export_type == "gguf":
success, message = backend.export_gguf(
save_directory = cmd.get("save_directory", ""),
quantization_method = cmd.get("quantization_method", "Q4_K_M"),
quantization_method = cmd.get("quantization_method", ["Q4_K_M"]),
push_to_hub = cmd.get("push_to_hub", False),
repo_id = cmd.get("repo_id"),
hf_token = cmd.get("hf_token"),

View file

@ -6,7 +6,7 @@ Pydantic schemas for Export API.
"""
from pydantic import BaseModel, Field
from typing import List, Optional, Literal, Dict, Any
from typing import Any, Dict, List, Literal, Optional, Set, Union
class LoadCheckpointRequest(BaseModel):
@ -108,9 +108,12 @@ class ExportGGUFRequest(BaseModel):
...,
description = "Directory where GGUF files will be saved",
)
quantization_method: str = Field(
quantization_method: Union[str, List[str]] = Field(
"Q4_K_M",
description = 'GGUF quantization method (e.g. "Q4_K_M")',
description = (
"GGUF quantization method, or list of methods to produce in a "
'single batch (e.g. "Q4_K_M" or ["Q4_K_M", "BF16"]).'
),
)
push_to_hub: bool = Field(
False,
@ -130,3 +133,39 @@ class ExportLoRAAdapterRequest(ExportCommonOptions):
"""Request for exporting only the LoRA adapter (not merged)."""
# Uses fields from ExportCommonOptions only
def normalize_gguf_quantization_method(
value: Union[str, List[str]],
) -> List[str]:
"""
Normalize a GGUF `quantization_method` value to a lowercase, deduplicated
list suitable for `save_pretrained_gguf`.
Accepts either a single string (legacy single-format callers) or a list
of strings (batch callers). Returns a list preserving first-seen order.
Raises:
ValueError: if the resulting list is empty. The route handler catches
this and returns `HTTPException(status_code=400, ...)` so clients
get a clear 400 instead of a generic 500 from deeper in the stack.
"""
if isinstance(value, str):
methods = [value]
else:
methods = list(value)
seen: Set[str] = set()
normalized: List[str] = []
for method in methods:
lowered = method.lower()
if lowered not in seen:
seen.add(lowered)
normalized.append(lowered)
if not normalized:
raise ValueError(
"quantization_method must contain at least one format",
)
return normalized

View file

@ -38,6 +38,7 @@ from models import (
ExportGGUFRequest,
ExportLoRAAdapterRequest,
)
from models.export import normalize_gguf_quantization_method
router = APIRouter()
logger = get_logger(__name__)
@ -253,11 +254,18 @@ async def export_gguf(
Wraps ExportBackend.export_gguf.
"""
try:
normalized_methods = normalize_gguf_quantization_method(
request.quantization_method,
)
except ValueError as exc:
raise HTTPException(status_code = 400, detail = str(exc))
try:
backend = get_export_backend()
success, message = backend.export_gguf(
save_directory = request.save_directory,
quantization_method = request.quantization_method,
quantization_method = normalized_methods,
push_to_hub = request.push_to_hub,
repo_id = request.repo_id,
hf_token = request.hf_token,

View file

@ -0,0 +1,98 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""
Unit tests for the GGUF export quantization_method normalization helper.
Tests the pure helper function `normalize_gguf_quantization_method` in
isolation no FastAPI app, no TestClient, no route invocation, no stubs.
The helper owns the Union[str, List[str]] coercion, case normalization,
deduplication, and empty-list rejection that `routes/export.py` relies on.
Corresponds to the design spec at
`.claude/specs/2026-04-14-studio-batch-gguf-exports-design.md`.
No GPU, no network, no heavy imports runs in milliseconds.
"""
from __future__ import annotations
import sys
from pathlib import Path
import pytest
# Match conftest.py's sys.path handling so flat imports from the backend
# root (e.g. `from models.export import ...`) resolve.
_backend_root = Path(__file__).resolve().parent.parent
if str(_backend_root) not in sys.path:
sys.path.insert(0, str(_backend_root))
from models.export import (
ExportGGUFRequest,
normalize_gguf_quantization_method,
)
class TestNormalizeGGUFQuantizationMethod:
"""Covers the five normalization contracts the route handler relies on."""
def test_string_input_wraps_and_lowercases(self):
"""A single string becomes a single-element lowercase list."""
assert normalize_gguf_quantization_method("Q4_K_M") == ["q4_k_m"]
def test_list_input_lowercases_each_element(self):
"""A list of strings is lowercased element-wise, order preserved."""
assert normalize_gguf_quantization_method(["Q4_K_M", "BF16"]) == [
"q4_k_m",
"bf16",
]
def test_duplicate_formats_deduped_preserving_first_seen_order(self):
"""Dedup runs after lowercasing; first occurrence wins the position."""
result = normalize_gguf_quantization_method(
["Q4_K_M", "q4_k_m", "Q8_0", "Q4_K_M"],
)
assert result == ["q4_k_m", "q8_0"]
def test_empty_list_raises_valueerror(self):
"""Empty input is rejected so the route can map it to HTTP 400."""
with pytest.raises(ValueError, match = "at least one"):
normalize_gguf_quantization_method([])
def test_single_element_list_passes_through(self):
"""Single-element list is a valid input (equivalent to the string form)."""
assert normalize_gguf_quantization_method(["BF16"]) == ["bf16"]
class TestExportGGUFRequestDefault:
"""Integration between Pydantic default and the normalization helper."""
def test_pydantic_default_normalizes_to_lowercase_list(self):
"""When the caller omits quantization_method, the default string flows
through the helper and becomes a one-element lowercase list."""
req = ExportGGUFRequest(save_directory = "/tmp/x")
assert normalize_gguf_quantization_method(req.quantization_method) == [
"q4_k_m",
]
def test_pydantic_accepts_string_input(self):
"""Legacy scripted callers can still send a single-format string."""
req = ExportGGUFRequest(
save_directory = "/tmp/x",
quantization_method = "Q4_K_M",
)
assert normalize_gguf_quantization_method(req.quantization_method) == [
"q4_k_m",
]
def test_pydantic_accepts_list_input(self):
"""New callers send the list form."""
req = ExportGGUFRequest(
save_directory = "/tmp/x",
quantization_method = ["Q4_K_M", "BF16"],
)
assert normalize_gguf_quantization_method(req.quantization_method) == [
"q4_k_m",
"bf16",
]

View file

@ -99,7 +99,7 @@ export async function exportBase(params: {
export async function exportGGUF(params: {
save_directory: string;
quantization_method: string;
quantization_method: string[];
push_to_hub?: boolean;
repo_id?: string | null;
hf_token?: string | null;

View file

@ -454,15 +454,13 @@ export function ExportPage() {
});
}
} else if (exportMethod === "gguf") {
for (const quant of quantLevels) {
await exportGGUF({
save_directory: saveDir,
quantization_method: quant,
push_to_hub: pushToHub,
repo_id: repoId,
hf_token: token,
});
}
await exportGGUF({
save_directory: saveDir,
quantization_method: quantLevels,
push_to_hub: pushToHub,
repo_id: repoId,
hf_token: token,
});
} else if (exportMethod === "lora") {
await exportLoRA({
save_directory: saveDir,