From 2f985ccbb51c5a7ab9bfb9a6f112ca33b25eaa91 Mon Sep 17 00:00:00 2001
From: Roland Tannous <115670425+rolandtannous@users.noreply.github.com>
Date: Tue, 24 Feb 2026 17:40:05 +0400
Subject: [PATCH 01/12] Add GGUF model inference via llama-server backend

---
 .gitignore                                    |   3 +
 setup.sh                                      |  62 ++-
 studio/backend/core/inference/__init__.py     |   2 +
 studio/backend/core/inference/llama_cpp.py    | 414 ++++++++++++++++++
 studio/backend/models/inference.py            |   2 +
 studio/backend/models/models.py               |   1 +
 studio/backend/routes/inference.py            | 268 ++++++++++--
 studio/backend/routes/models.py               |  18 +
 studio/backend/utils/models/model_config.py   |  50 ++-
 .../chat/hooks/use-chat-model-runtime.ts      |   6 +-
 .../frontend/src/features/chat/types/api.ts   |   3 +
 .../src/features/chat/types/runtime.ts        |   1 +
 .../frontend/src/hooks/use-hf-model-search.ts |   1 -
 13 files changed, 791 insertions(+), 40 deletions(-)
 create mode 100644 studio/backend/core/inference/llama_cpp.py

diff --git a/.gitignore b/.gitignore
index 3901c1c699..e24c38c2b0 100755
--- a/.gitignore
+++ b/.gitignore
@@ -24,6 +24,9 @@ unsloth_training_checkpoints/
 *.gguf
 *.safetensors
 
+# Built binaries (llama-server etc.)
+bin/
+
 # IDE / Editors
 .vscode/
 .idea/
diff --git a/setup.sh b/setup.sh
index 1f55e141fd..be17370df4 100755
--- a/setup.sh
+++ b/setup.sh
@@ -206,7 +206,67 @@ else
     fi
 fi
 
-# ── 8. Add shell alias (skip in Colab) ──
+# ── 8. Build llama-server for GGUF inference ──
+# Builds in an isolated temp directory to avoid conflicts with unsloth-zoo's
+# own llama.cpp management (used for GGUF export). Only the llama-server
+# binary is extracted to $REPO/bin/.
+LLAMA_SERVER_BIN="$SCRIPT_DIR/bin/llama-server"
+if [ -f "$LLAMA_SERVER_BIN" ]; then
+    echo ""
+    echo "✅ llama-server already exists at $LLAMA_SERVER_BIN"
+else
+    # Check prerequisites
+    if ! command -v cmake &>/dev/null; then
+        echo ""
+        echo "⚠️  cmake not found — skipping llama-server build (GGUF inference won't be available)"
+        echo "   Install cmake and re-run setup.sh to enable GGUF inference."
+    elif ! command -v git &>/dev/null; then
+        echo ""
+        echo "⚠️  git not found — skipping llama-server build (GGUF inference won't be available)"
+    else
+        echo ""
+        echo "Building llama-server for GGUF inference..."
+        LLAMA_BUILD_TMP=$(mktemp -d)
+
+        BUILD_OK=true
+        run_quiet "clone llama.cpp" git clone --depth 1 https://github.com/ggml-org/llama.cpp.git "$LLAMA_BUILD_TMP/llama.cpp" || BUILD_OK=false
+
+        if [ "$BUILD_OK" = true ]; then
+            CMAKE_ARGS=""
+            if command -v nvcc &>/dev/null; then
+                echo "   Building with CUDA support..."
+                CMAKE_ARGS="-DGGML_CUDA=ON"
+            else
+                echo "   Building CPU-only (no CUDA detected)..."
+            fi
+
+            NCPU=$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4)
+
+            run_quiet "cmake llama.cpp" cmake -S "$LLAMA_BUILD_TMP/llama.cpp" -B "$LLAMA_BUILD_TMP/llama.cpp/build" $CMAKE_ARGS || BUILD_OK=false
+        fi
+
+        if [ "$BUILD_OK" = true ]; then
+            run_quiet "build llama-server" cmake --build "$LLAMA_BUILD_TMP/llama.cpp/build" --config Release --target llama-server -j"$NCPU" || BUILD_OK=false
+        fi
+
+        if [ "$BUILD_OK" = true ]; then
+            mkdir -p "$SCRIPT_DIR/bin"
+            if [ -f "$LLAMA_BUILD_TMP/llama.cpp/build/bin/llama-server" ]; then
+                cp "$LLAMA_BUILD_TMP/llama.cpp/build/bin/llama-server" "$LLAMA_SERVER_BIN"
+                echo "✅ llama-server built and installed to $LLAMA_SERVER_BIN"
+            else
+                echo "⚠️  llama-server binary not found after build — GGUF inference won't be available"
+            fi
+        else
+            echo "⚠️  llama-server build failed — GGUF inference won't be available, but everything else works"
+        fi
+
+        # Clean up temp build directory
+        rm -rf "$LLAMA_BUILD_TMP"
+    fi
+fi
+
+# ── 9. Add shell alias (skip in Colab) ──
 # Note: venv activation does NOT persist across terminal sessions.
 # This alias hardcodes the venv python path so users don't need to activate.
 if [ "$IS_COLAB" = false ]; then
diff --git a/studio/backend/core/inference/__init__.py b/studio/backend/core/inference/__init__.py
index 494229a087..ff8b75d36a 100644
--- a/studio/backend/core/inference/__init__.py
+++ b/studio/backend/core/inference/__init__.py
@@ -2,8 +2,10 @@
 Inference submodule - Inference backend for model loading and generation
 """
 from .inference import InferenceBackend, get_inference_backend
+from .llama_cpp import LlamaCppBackend
 
 __all__ = [
     'InferenceBackend',
     'get_inference_backend',
+    'LlamaCppBackend',
 ]
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
new file mode 100644
index 0000000000..dae8d20489
--- /dev/null
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -0,0 +1,414 @@
+"""
+llama-server inference backend for GGUF models.
+
+Manages a llama-server subprocess and proxies chat completions
+through its /v1/completions endpoint.
+"""
+import atexit
+import json
+import logging
+import shutil
+import signal
+import socket
+import subprocess
+import threading
+import time
+from pathlib import Path
+from typing import Generator, Optional
+
+import httpx
+
+logger = logging.getLogger(__name__)
+
+
+class LlamaCppBackend:
+    """
+    Manages a llama-server subprocess for GGUF model inference.
+
+    Lifecycle:
+        1. load_model()  — starts llama-server with the GGUF file
+        2. generate_chat_completion() — formats prompt, proxies to /v1/completions, streams back
+        3. unload_model() — terminates llama-server subprocess
+    """
+
+    def __init__(self):
+        self._process: Optional[subprocess.Popen] = None
+        self._port: Optional[int] = None
+        self._model_identifier: Optional[str] = None
+        self._gguf_path: Optional[str] = None
+        self._healthy = False
+        self._lock = threading.Lock()
+        self._chat_template: Optional[str] = None
+
+        atexit.register(self._cleanup)
+
+    # ── Properties ────────────────────────────────────────────────
+
+    @property
+    def is_loaded(self) -> bool:
+        return self._process is not None and self._healthy
+
+    @property
+    def base_url(self) -> str:
+        return f"http://127.0.0.1:{self._port}"
+
+    @property
+    def model_identifier(self) -> Optional[str]:
+        return self._model_identifier
+
+    # ── Binary discovery ──────────────────────────────────────────
+
+    @staticmethod
+    def _find_llama_server_binary() -> Optional[str]:
+        """
+        Locate the llama-server binary.
+
+        Search order:
+        1. LLAMA_SERVER_PATH environment variable
+        2. ./bin/llama-server  (built by setup.sh)
+        3. llama-server on PATH  (system install)
+        4. ./llama.cpp/llama-server  (unsloth-zoo build output)
+        """
+        import os
+
+        # 1. Env var
+        env_path = os.environ.get("LLAMA_SERVER_PATH")
+        if env_path and Path(env_path).is_file():
+            return env_path
+
+        # 2. Project bin/ directory (setup.sh output)
+        project_root = Path(__file__).resolve().parents[3]  # core/inference/ → backend/ → studio/ → root
+        bin_path = project_root / "bin" / "llama-server"
+        if bin_path.is_file():
+            return str(bin_path)
+
+        # 3. System PATH
+        system_path = shutil.which("llama-server")
+        if system_path:
+            return system_path
+
+        # 4. unsloth-zoo build output (from GGUF export)
+        llama_cpp_path = project_root / "llama.cpp" / "llama-server"
+        if llama_cpp_path.is_file():
+            return str(llama_cpp_path)
+
+        return None
+
+    # ── Port allocation ───────────────────────────────────────────
+
+    @staticmethod
+    def _find_free_port() -> int:
+        """Find an available TCP port."""
+        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+            s.bind(("", 0))
+            return s.getsockname()[1]
+
+    # ── Lifecycle ─────────────────────────────────────────────────
+
+    def load_model(
+        self,
+        gguf_path: str,
+        model_identifier: str,
+        n_ctx: int = 4096,
+        n_gpu_layers: int = -1,
+        n_threads: Optional[int] = None,
+    ) -> bool:
+        """
+        Start llama-server with the given GGUF file.
+
+        Args:
+            gguf_path: Path to the .gguf file
+            model_identifier: Display identifier for the model
+            n_ctx: Context window size
+            n_gpu_layers: Number of layers to offload to GPU (-1 = all)
+            n_threads: Number of CPU threads (None = auto)
+
+        Returns:
+            True if server started and health check passed.
+        """
+        with self._lock:
+            # Kill existing process if any
+            self._kill_process()
+
+            binary = self._find_llama_server_binary()
+            if not binary:
+                raise RuntimeError(
+                    "llama-server binary not found. "
+                    "Run setup.sh to build it, install llama.cpp, "
+                    "or set LLAMA_SERVER_PATH environment variable."
+                )
+
+            if not Path(gguf_path).is_file():
+                raise FileNotFoundError(f"GGUF file not found: {gguf_path}")
+
+            self._port = self._find_free_port()
+            cmd = [
+                binary,
+                "-m", gguf_path,
+                "--port", str(self._port),
+                "-c", str(n_ctx),
+                "-ngl", str(n_gpu_layers),
+            ]
+            if n_threads is not None:
+                cmd.extend(["--threads", str(n_threads)])
+
+            logger.info(f"Starting llama-server: {' '.join(cmd)}")
+
+            self._process = subprocess.Popen(
+                cmd,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,
+                text=True,
+            )
+
+            self._gguf_path = gguf_path
+            self._model_identifier = model_identifier
+
+            # Wait for health
+            if not self._wait_for_health(timeout=120.0):
+                self._kill_process()
+                raise RuntimeError(
+                    "llama-server failed to start. "
+                    "Check that the GGUF file is valid and you have enough memory."
+                )
+
+            self._healthy = True
+
+            # Try to read chat template from GGUF metadata
+            self._chat_template = self._read_gguf_chat_template(gguf_path)
+
+            logger.info(
+                f"llama-server ready on port {self._port} "
+                f"for model '{model_identifier}'"
+            )
+            return True
+
+    def unload_model(self) -> bool:
+        """Terminate the llama-server subprocess and clean up state."""
+        with self._lock:
+            self._kill_process()
+            logger.info(f"Unloaded GGUF model: {self._model_identifier}")
+            self._model_identifier = None
+            self._gguf_path = None
+            self._port = None
+            self._healthy = False
+            self._chat_template = None
+            return True
+
+    def _kill_process(self):
+        """Terminate the subprocess if running."""
+        if self._process is None:
+            return
+        try:
+            self._process.terminate()
+            self._process.wait(timeout=5)
+        except subprocess.TimeoutExpired:
+            logger.warning("llama-server did not exit on SIGTERM, sending SIGKILL")
+            self._process.kill()
+            self._process.wait(timeout=5)
+        except Exception as e:
+            logger.warning(f"Error killing llama-server process: {e}")
+        finally:
+            self._process = None
+
+    def _cleanup(self):
+        """atexit handler to ensure llama-server is terminated."""
+        self._kill_process()
+
+    def _wait_for_health(self, timeout: float = 120.0, interval: float = 0.5) -> bool:
+        """
+        Poll llama-server's /health endpoint until it responds 200.
+
+        Also monitors subprocess for early exit/crash.
+        """
+        deadline = time.monotonic() + timeout
+        url = f"http://127.0.0.1:{self._port}/health"
+
+        while time.monotonic() < deadline:
+            # Check if process crashed
+            if self._process.poll() is not None:
+                # Read remaining output for error info
+                output = self._process.stdout.read() if self._process.stdout else ""
+                logger.error(
+                    f"llama-server exited with code {self._process.returncode}. "
+                    f"Output: {output[:2000]}"
+                )
+                return False
+
+            try:
+                resp = httpx.get(url, timeout=2.0)
+                if resp.status_code == 200:
+                    return True
+            except (httpx.ConnectError, httpx.TimeoutException):
+                pass
+
+            time.sleep(interval)
+
+        logger.error(f"llama-server health check timed out after {timeout}s")
+        return False
+
+    # ── Chat template ─────────────────────────────────────────────
+
+    @staticmethod
+    def _read_gguf_chat_template(gguf_path: str) -> Optional[str]:
+        """
+        Try to read the chat_template from GGUF file metadata.
+
+        Uses the gguf Python library if available.
+        Returns the Jinja2 template string, or None.
+        """
+        try:
+            from gguf import GGUFReader
+
+            reader = GGUFReader(gguf_path)
+            for field_name in reader.fields:
+                if field_name == "tokenizer.chat_template":
+                    field = reader.fields[field_name]
+                    # Field data is an array of bytes
+                    template_bytes = bytes(field.parts[field.data[0]])
+                    template = template_bytes.decode("utf-8")
+                    logger.info(f"Read chat template from GGUF metadata ({len(template)} chars)")
+                    return template
+        except ImportError:
+            logger.debug("gguf library not available, cannot read chat template from GGUF metadata")
+        except Exception as e:
+            logger.warning(f"Could not read chat template from GGUF: {e}")
+
+        return None
+
+    def format_prompt(self, messages: list[dict], system_prompt: str = "") -> str:
+        """
+        Format chat messages into a raw prompt string for /v1/completions.
+
+        Attempts to:
+        1. Render the GGUF's embedded chat_template with Jinja2
+        2. Fallback to ChatML format
+        """
+        # Build full message list with system prompt
+        full_messages = []
+        if system_prompt:
+            full_messages.append({"role": "system", "content": system_prompt})
+        full_messages.extend(messages)
+
+        # Try Jinja2 rendering if we have a template
+        if self._chat_template:
+            try:
+                return self._render_jinja_template(full_messages)
+            except Exception as e:
+                logger.warning(f"Jinja2 template rendering failed, falling back to ChatML: {e}")
+
+        # Fallback: ChatML format
+        return self._format_chatml(full_messages)
+
+    def _render_jinja_template(self, messages: list[dict]) -> str:
+        """Render messages using the GGUF's Jinja2 chat template."""
+        from jinja2 import BaseLoader, Environment
+
+        env = Environment(loader=BaseLoader(), keep_trailing_newline=True)
+        # Add common template globals
+        env.globals["raise_exception"] = lambda msg: (_ for _ in ()).throw(ValueError(msg))
+
+        template = env.from_string(self._chat_template)
+        rendered = template.render(
+            messages=messages,
+            add_generation_prompt=True,
+            bos_token="<s>",
+            eos_token="</s>",
+        )
+        return rendered
+
+    @staticmethod
+    def _format_chatml(messages: list[dict]) -> str:
+        """Format messages using ChatML template (universal fallback)."""
+        parts = []
+        for msg in messages:
+            role = msg.get("role", "user")
+            content = msg.get("content", "")
+            parts.append(f"<|im_start|>{role}\n{content}<|im_end|>")
+        parts.append("<|im_start|>assistant")
+        return "\n".join(parts) + "\n"
+
+    # ── Generation (proxy to llama-server) ────────────────────────
+
+    def generate_chat_completion(
+        self,
+        prompt: str,
+        temperature: float = 0.7,
+        top_p: float = 0.9,
+        top_k: int = 40,
+        min_p: float = 0.0,
+        max_tokens: int = 512,
+        repetition_penalty: float = 1.1,
+        stop: Optional[list[str]] = None,
+        cancel_event: Optional[threading.Event] = None,
+    ) -> Generator[str, None, None]:
+        """
+        Send a completion request to llama-server and stream tokens back.
+
+        Uses /v1/completions (NOT /v1/chat/completions) so we control
+        the prompt format entirely.
+
+        Yields cumulative text (matching InferenceBackend's convention).
+        """
+        if not self.is_loaded:
+            raise RuntimeError("llama-server is not loaded")
+
+        payload = {
+            "prompt": prompt,
+            "stream": True,
+            "temperature": temperature,
+            "top_p": top_p,
+            "top_k": top_k if top_k >= 0 else 0,
+            "min_p": min_p,
+            "n_predict": max_tokens,
+            "repeat_penalty": repetition_penalty,
+        }
+        if stop:
+            payload["stop"] = stop
+
+        url = f"{self.base_url}/v1/completions"
+        cumulative = ""
+
+        try:
+            with httpx.Client(timeout=None) as client:
+                with client.stream("POST", url, json=payload) as response:
+                    if response.status_code != 200:
+                        error_body = response.read().decode()
+                        raise RuntimeError(
+                            f"llama-server returned {response.status_code}: {error_body}"
+                        )
+
+                    buffer = ""
+                    for raw_chunk in response.iter_text():
+                        if cancel_event is not None and cancel_event.is_set():
+                            break
+
+                        buffer += raw_chunk
+                        while "\n" in buffer:
+                            line, buffer = buffer.split("\n", 1)
+                            line = line.strip()
+
+                            if not line:
+                                continue
+                            if line == "data: [DONE]":
+                                return
+                            if not line.startswith("data: "):
+                                continue
+
+                            try:
+                                data = json.loads(line[6:])
+                                choices = data.get("choices", [])
+                                if choices:
+                                    token = choices[0].get("text", "")
+                                    if token:
+                                        cumulative += token
+                                        yield cumulative
+                            except json.JSONDecodeError:
+                                logger.debug(f"Skipping malformed SSE line: {line[:100]}")
+
+        except httpx.ConnectError:
+            raise RuntimeError("Lost connection to llama-server")
+        except Exception as e:
+            if cancel_event is not None and cancel_event.is_set():
+                return
+            raise
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py
index d2d98d7944..c4a062ae3a 100644
--- a/studio/backend/models/inference.py
+++ b/studio/backend/models/inference.py
@@ -43,6 +43,7 @@ class LoadResponse(BaseModel):
     display_name: str = Field(..., description="Display name of the model")
     is_vision: bool = Field(False, description="Whether model is a vision model")
     is_lora: bool = Field(False, description="Whether model is a LoRA adapter")
+    is_gguf: bool = Field(False, description="Whether model is a GGUF model (llama.cpp)")
     inference: dict = Field(..., description="Inference parameters (temperature, top_p, top_k, min_p)")
 
 
@@ -56,6 +57,7 @@ class InferenceStatusResponse(BaseModel):
     """Current inference backend status"""
     active_model: Optional[str] = Field(None, description="Currently active model identifier")
     is_vision: bool = Field(False, description="Whether the active model is a vision model")
+    is_gguf: bool = Field(False, description="Whether the active model is a GGUF model (llama.cpp)")
     loading: List[str] = Field(default_factory=list, description="Models currently being loaded")
     loaded: List[str] = Field(default_factory=list, description="Models currently loaded")
 
diff --git a/studio/backend/models/models.py b/studio/backend/models/models.py
index 5542db76d5..b3f9b50ed2 100644
--- a/studio/backend/models/models.py
+++ b/studio/backend/models/models.py
@@ -53,6 +53,7 @@ class ModelDetails(BaseModel):
     config: Optional[Dict[str, Any]] = Field(None, description="Model configuration dictionary")
     is_vision: bool = Field(False, description="Whether model is a vision model")
     is_lora: bool = Field(False, description="Whether model is a LoRA adapter")
+    is_gguf: bool = Field(False, description="Whether model is a GGUF model (llama.cpp format)")
     base_model: Optional[str] = Field(None, description="Base model if this is a LoRA adapter")
 
 
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index c30a1638d4..156578b960 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -23,6 +23,7 @@ if str(backend_path) not in sys.path:
 # Import backend functions
 try:
     from core.inference import get_inference_backend
+    from core.inference.llama_cpp import LlamaCppBackend
     from utils.models import ModelConfig
     from utils.inference import load_inference_config
 except ImportError:
@@ -30,6 +31,7 @@ except ImportError:
     if str(parent_backend) not in sys.path:
         sys.path.insert(0, str(parent_backend))
     from core.inference import get_inference_backend
+    from core.inference.llama_cpp import LlamaCppBackend
     from utils.models import ModelConfig
     from utils.inference import load_inference_config
 
@@ -61,60 +63,111 @@ if not logger.handlers:
     logger.addHandler(handler)
     logger.setLevel(logging.INFO)
 
+# GGUF inference backend (llama-server)
+_llama_cpp_backend = LlamaCppBackend()
+
+def get_llama_cpp_backend() -> LlamaCppBackend:
+    return _llama_cpp_backend
+
 
 @router.post("/load", response_model=LoadResponse)
 async def load_model(request: LoadRequest):
     """
     Load a model for inference.
-    
+
     The model_path should be a clean identifier from GET /models/list.
     Returns inference configuration parameters (temperature, top_p, top_k, min_p)
     from the model's YAML config, falling back to default.yaml for missing values.
+
+    GGUF models are loaded via llama-server (llama.cpp) instead of Unsloth.
     """
     try:
-        backend = get_inference_backend()
-        
         # Create config using clean factory method
         # is_lora is auto-detected from adapter_config.json on disk/HF
         config = ModelConfig.from_identifier(
             model_id=request.model_path,
             hf_token=request.hf_token,
         )
-        
+
         if not config:
             raise HTTPException(
                 status_code=400,
                 detail=f"Invalid model identifier: {request.model_path}"
             )
-        
-        # Load the model
+
+        # ── GGUF path: load via llama-server ──────────────────────
+        if config.is_gguf:
+            llama_backend = get_llama_cpp_backend()
+            unsloth_backend = get_inference_backend()
+
+            # Unload any active Unsloth model first to free VRAM
+            if unsloth_backend.active_model_name:
+                logger.info(f"Unloading Unsloth model '{unsloth_backend.active_model_name}' before loading GGUF")
+                unsloth_backend.unload_model(unsloth_backend.active_model_name)
+
+            success = llama_backend.load_model(
+                gguf_path=config.gguf_file,
+                model_identifier=config.identifier,
+                n_ctx=request.max_seq_length,
+            )
+
+            if not success:
+                raise HTTPException(
+                    status_code=500,
+                    detail=f"Failed to load GGUF model: {config.display_name}"
+                )
+
+            logger.info(f"Loaded GGUF model via llama-server: {config.identifier}")
+
+            inference_config = load_inference_config(config.identifier)
+
+            return LoadResponse(
+                status="loaded",
+                model=config.identifier,
+                display_name=config.display_name,
+                is_vision=False,
+                is_lora=False,
+                is_gguf=True,
+                inference=inference_config,
+            )
+
+        # ── Standard path: load via Unsloth/transformers ──────────
+        backend = get_inference_backend()
+
+        # Unload any active GGUF model first
+        llama_backend = get_llama_cpp_backend()
+        if llama_backend.is_loaded:
+            logger.info("Unloading GGUF model before loading Unsloth model")
+            llama_backend.unload_model()
+
         success = backend.load_model(
             config=config,
             max_seq_length=request.max_seq_length,
             load_in_4bit=request.load_in_4bit,
             hf_token=request.hf_token,
         )
-        
+
         if not success:
             raise HTTPException(
                 status_code=500,
                 detail=f"Failed to load model: {config.display_name}"
             )
-        
+
         logger.info(f"Loaded model: {config.identifier}")
-        
+
         # Load inference configuration parameters
         inference_config = load_inference_config(config.identifier)
-        
+
         return LoadResponse(
             status="loaded",
             model=config.identifier,
             display_name=config.display_name,
             is_vision=config.is_vision,
             is_lora=config.is_lora,
+            is_gguf=False,
             inference=inference_config,
         )
-        
+
     except HTTPException:
         raise
     except Exception as e:
@@ -129,13 +182,22 @@ async def load_model(request: LoadRequest):
 async def unload_model(request: UnloadRequest):
     """
     Unload a model from memory.
+    Routes to the correct backend (llama-server for GGUF, Unsloth otherwise).
     """
     try:
+        # Check if the GGUF backend has this model loaded
+        llama_backend = get_llama_cpp_backend()
+        if llama_backend.is_loaded and llama_backend.model_identifier == request.model_path:
+            llama_backend.unload_model()
+            logger.info(f"Unloaded GGUF model: {request.model_path}")
+            return UnloadResponse(status="unloaded", model=request.model_path)
+
+        # Otherwise, unload from Unsloth backend
         backend = get_inference_backend()
         backend.unload_model(request.model_path)
         logger.info(f"Unloaded model: {request.model_path}")
         return UnloadResponse(status="unloaded", model=request.model_path)
-        
+
     except Exception as e:
         logger.error(f"Error unloading model: {e}", exc_info=True)
         raise HTTPException(
@@ -221,22 +283,37 @@ async def generate_stream(request: GenerateRequest):
 async def get_status():
     """
     Get current inference backend status.
+    Reports whichever backend (Unsloth or llama-server) is currently active.
     """
     try:
+        llama_backend = get_llama_cpp_backend()
+
+        # If a GGUF model is loaded via llama-server, report that
+        if llama_backend.is_loaded:
+            return InferenceStatusResponse(
+                active_model=llama_backend.model_identifier,
+                is_vision=False,
+                is_gguf=True,
+                loading=[],
+                loaded=[llama_backend.model_identifier],
+            )
+
+        # Otherwise, report Unsloth backend status
         backend = get_inference_backend()
-        
+
         is_vision = False
         if backend.active_model_name:
             model_info = backend.models.get(backend.active_model_name, {})
             is_vision = model_info.get("is_vision", False)
-        
+
         return InferenceStatusResponse(
             active_model=backend.active_model_name,
             is_vision=is_vision,
+            is_gguf=False,
             loading=list(getattr(backend, 'loading_models', set())),
             loaded=list(backend.models.keys()),
         )
-        
+
     except Exception as e:
         logger.error(f"Error getting status: {e}", exc_info=True)
         raise HTTPException(
@@ -315,29 +392,157 @@ async def openai_chat_completions(payload: ChatCompletionRequest, request: Reque
 
     Streaming (default):  returns SSE chunks matching OpenAI's format.
     Non-streaming:        returns a single ChatCompletion JSON object.
-    """
-    backend = get_inference_backend()
 
-    if not backend.active_model_name:
-        raise HTTPException(
-            status_code=400,
-            detail="No model loaded. Call POST /inference/load first.",
-        )
+    Automatically routes to the correct backend:
+    - GGUF models → llama-server via LlamaCppBackend
+    - Other models → Unsloth/transformers via InferenceBackend
+    """
+    llama_backend = get_llama_cpp_backend()
+    using_gguf = llama_backend.is_loaded
+
+    # ── Determine which backend is active ─────────────────────
+    if using_gguf:
+        model_name = llama_backend.model_identifier or payload.model
+    else:
+        backend = get_inference_backend()
+        if not backend.active_model_name:
+            raise HTTPException(
+                status_code=400,
+                detail="No model loaded. Call POST /inference/load first.",
+            )
+        model_name = backend.active_model_name or payload.model
 
     # ── Parse messages (handles multimodal content parts) ─────
     system_prompt, chat_messages, extracted_image_b64 = _extract_content_parts(
         payload.messages
     )
 
-    # If no non-system messages were provided, error out
     if not chat_messages:
         raise HTTPException(
             status_code=400,
             detail="At least one non-system message is required.",
         )
 
-    # ── Decode image (from content parts OR legacy field) ─────
-    # Content-part images take priority; fall back to legacy field
+    # ── GGUF path: format prompt → proxy to llama-server ──────
+    if using_gguf:
+        # GGUF models don't support vision
+        image_b64 = extracted_image_b64 or payload.image_base64
+        if image_b64:
+            raise HTTPException(
+                status_code=400,
+                detail="Image provided but GGUF models do not support vision.",
+            )
+
+        prompt = llama_backend.format_prompt(chat_messages, system_prompt)
+        cancel_event = threading.Event()
+
+        completion_id = f"chatcmpl-{uuid.uuid4().hex[:12]}"
+        created = int(time.time())
+
+        def gguf_generate():
+            return llama_backend.generate_chat_completion(
+                prompt=prompt,
+                temperature=payload.temperature,
+                top_p=payload.top_p,
+                top_k=payload.top_k,
+                min_p=payload.min_p,
+                max_tokens=payload.max_tokens or 512,
+                repetition_penalty=payload.repetition_penalty,
+                cancel_event=cancel_event,
+            )
+
+        if payload.stream:
+            async def gguf_stream_chunks():
+                try:
+                    # First chunk: role
+                    first_chunk = ChatCompletionChunk(
+                        id=completion_id,
+                        created=created,
+                        model=model_name,
+                        choices=[ChunkChoice(
+                            delta=ChoiceDelta(role="assistant"),
+                            finish_reason=None,
+                        )],
+                    )
+                    yield f"data: {first_chunk.model_dump_json(exclude_none=True)}\n\n"
+
+                    # Content chunks — llama backend yields cumulative text
+                    prev_text = ""
+                    for cumulative in gguf_generate():
+                        if await request.is_disconnected():
+                            cancel_event.set()
+                            return
+                        new_text = cumulative[len(prev_text):]
+                        prev_text = cumulative
+                        if not new_text:
+                            continue
+                        chunk = ChatCompletionChunk(
+                            id=completion_id,
+                            created=created,
+                            model=model_name,
+                            choices=[ChunkChoice(
+                                delta=ChoiceDelta(content=new_text),
+                                finish_reason=None,
+                            )],
+                        )
+                        yield f"data: {chunk.model_dump_json(exclude_none=True)}\n\n"
+
+                    # Final chunk
+                    final_chunk = ChatCompletionChunk(
+                        id=completion_id,
+                        created=created,
+                        model=model_name,
+                        choices=[ChunkChoice(
+                            delta=ChoiceDelta(),
+                            finish_reason="stop",
+                        )],
+                    )
+                    yield f"data: {final_chunk.model_dump_json(exclude_none=True)}\n\n"
+                    yield "data: [DONE]\n\n"
+
+                except asyncio.CancelledError:
+                    cancel_event.set()
+                    raise
+                except Exception as e:
+                    logger.error(f"Error during GGUF streaming: {e}", exc_info=True)
+                    error_chunk = {
+                        "error": {"message": str(e), "type": "server_error"},
+                    }
+                    yield f"data: {json.dumps(error_chunk)}\n\n"
+
+            return StreamingResponse(
+                gguf_stream_chunks(),
+                media_type="text/event-stream",
+                headers={
+                    "Cache-Control": "no-cache",
+                    "Connection": "keep-alive",
+                    "X-Accel-Buffering": "no",
+                },
+            )
+        else:
+            try:
+                full_text = ""
+                for token in gguf_generate():
+                    full_text = token
+
+                response = ChatCompletion(
+                    id=completion_id,
+                    created=created,
+                    model=model_name,
+                    choices=[CompletionChoice(
+                        message=CompletionMessage(content=full_text),
+                        finish_reason="stop",
+                    )],
+                )
+                return JSONResponse(content=response.model_dump())
+
+            except Exception as e:
+                logger.error(f"Error during GGUF completion: {e}", exc_info=True)
+                raise HTTPException(status_code=500, detail=str(e))
+
+    # ── Standard Unsloth path ─────────────────────────────────
+
+    # Decode image (from content parts OR legacy field)
     image_b64 = extracted_image_b64 or payload.image_base64
     image = None
 
@@ -363,7 +568,7 @@ async def openai_chat_completions(payload: ChatCompletionRequest, request: Reque
         except Exception as e:
             raise HTTPException(status_code=400, detail=f"Failed to decode image: {e}")
 
-    # ── Shared generation kwargs ──────────────────────────────
+    # Shared generation kwargs
     gen_kwargs = dict(
         messages=chat_messages,
         system_prompt=system_prompt,
@@ -376,11 +581,10 @@ async def openai_chat_completions(payload: ChatCompletionRequest, request: Reque
         repetition_penalty=payload.repetition_penalty,
     )
 
-    # ── Choose generation path (adapter-controlled or standard) ──
+    # Choose generation path (adapter-controlled or standard)
     cancel_event = threading.Event()
 
     if payload.use_adapter is not None:
-        # Compare mode: toggle adapter state atomically with generation
         def generate():
             return backend.generate_with_adapter_control(
                 use_adapter=payload.use_adapter,
@@ -388,11 +592,9 @@ async def openai_chat_completions(payload: ChatCompletionRequest, request: Reque
                 **gen_kwargs,
             )
     else:
-        # Standard path: no adapter toggling
         def generate():
             return backend.generate_chat_response(cancel_event=cancel_event, **gen_kwargs)
 
-    model_name = backend.active_model_name or payload.model
     completion_id = f"chatcmpl-{uuid.uuid4().hex[:12]}"
     created = int(time.time())
 
@@ -400,7 +602,6 @@ async def openai_chat_completions(payload: ChatCompletionRequest, request: Reque
     if payload.stream:
         async def stream_chunks():
             try:
-                # First chunk: send the role
                 first_chunk = ChatCompletionChunk(
                     id=completion_id,
                     created=created,
@@ -412,8 +613,6 @@ async def openai_chat_completions(payload: ChatCompletionRequest, request: Reque
                 )
                 yield f"data: {first_chunk.model_dump_json(exclude_none=True)}\n\n"
 
-                # Content chunks — generate_chat_response yields cumulative
-                # text, so we diff to get incremental deltas.
                 prev_text = ""
                 for cumulative in generate():
                     if await request.is_disconnected():
@@ -435,7 +634,6 @@ async def openai_chat_completions(payload: ChatCompletionRequest, request: Reque
                     )
                     yield f"data: {chunk.model_dump_json(exclude_none=True)}\n\n"
 
-                # Final chunk: finish_reason = stop
                 final_chunk = ChatCompletionChunk(
                     id=completion_id,
                     created=created,
@@ -475,7 +673,7 @@ async def openai_chat_completions(payload: ChatCompletionRequest, request: Reque
         try:
             full_text = ""
             for token in generate():
-                full_text = token  # generate_stream yields cumulative text
+                full_text = token
 
             response = ChatCompletion(
                 id=completion_id,
diff --git a/studio/backend/routes/models.py b/studio/backend/routes/models.py
index 761c04d3e7..881f399c3d 100644
--- a/studio/backend/routes/models.py
+++ b/studio/backend/routes/models.py
@@ -88,6 +88,7 @@ def _scan_models_dir(models_dir: Path) -> List[LocalModelInfo]:
             or (child / "adapter_config.json").exists()
             or any(child.glob("*.safetensors"))
             or any(child.glob("*.bin"))
+            or any(child.glob("*.gguf"))
         )
         if not has_model_files:
             continue
@@ -104,6 +105,23 @@ def _scan_models_dir(models_dir: Path) -> List[LocalModelInfo]:
                 updated_at=updated_at,
             ),
         )
+    # Also scan for standalone .gguf files directly in the models directory
+    for gguf_file in models_dir.glob("*.gguf"):
+        if gguf_file.is_file():
+            try:
+                updated_at = gguf_file.stat().st_mtime
+            except OSError:
+                updated_at = None
+            found.append(
+                LocalModelInfo(
+                    id=str(gguf_file),
+                    display_name=gguf_file.stem,
+                    path=str(gguf_file),
+                    source="models_dir",
+                    updated_at=updated_at,
+                ),
+            )
+
     return found
 
 
diff --git a/studio/backend/utils/models/model_config.py b/studio/backend/utils/models/model_config.py
index fdf89fce39..27fbe3ee44 100644
--- a/studio/backend/utils/models/model_config.py
+++ b/studio/backend/utils/models/model_config.py
@@ -422,6 +422,32 @@ def is_vision_model(model_name: str, hf_token: Optional[str] = None) -> bool:
 pass
 
 
+def detect_gguf_model(path: str) -> Optional[str]:
+    """
+    Check if the given path is or contains a GGUF model file.
+
+    Handles three cases:
+    1. path is a direct .gguf file path
+    2. path is a directory containing .gguf files
+    3. path is an HuggingFace repo with GGUF files (not yet — future enhancement)
+
+    Returns the full path to the .gguf file if found, None otherwise.
+    """
+    p = Path(path)
+
+    # Case 1: direct .gguf file
+    if p.suffix == ".gguf" and p.is_file():
+        return str(p.resolve())
+
+    # Case 2: directory containing .gguf files
+    if p.is_dir():
+        gguf_files = sorted(p.glob("*.gguf"), key=lambda f: f.stat().st_size, reverse=True)
+        if gguf_files:
+            return str(gguf_files[0].resolve())
+
+    return None
+
+
 def scan_trained_loras(outputs_dir: str = "./outputs") -> List[Tuple[str, str]]:
     """
     Scan outputs folder for trained LoRA adapters.
@@ -595,6 +621,8 @@ class ModelConfig:
     is_cached: bool     # Is this already in HF cache?
     is_vision: bool     # Is this a vision model?
     is_lora: bool       # Is this a lora adapter?
+    is_gguf: bool = False       # Is this a GGUF model?
+    gguf_file: Optional[str] = None  # Full path to the .gguf file
     base_model: Optional[str] = None  # Base model (for LoRAs)
 
     @classmethod
@@ -675,12 +703,30 @@ class ModelConfig:
         identifier = model_id.strip()
         is_local = is_local_path(identifier)
         path = normalize_path(identifier) if is_local else identifier
-        
+
         # Add unsloth/ prefix for shorthand HF models
         if not is_local and "/" not in identifier:
             identifier = f"unsloth/{identifier}"
             path = identifier
-        
+
+        # Auto-detect GGUF models (check before LoRA/vision detection)
+        if is_local:
+            gguf_file = detect_gguf_model(path)
+            if gguf_file:
+                display_name = Path(gguf_file).stem
+                logger.info(f"Detected GGUF model: {gguf_file}")
+                return cls(
+                    identifier=identifier,
+                    display_name=display_name,
+                    path=path,
+                    is_local=True,
+                    is_cached=True,
+                    is_vision=False,
+                    is_lora=False,
+                    is_gguf=True,
+                    gguf_file=gguf_file,
+                )
+
         # Auto-detect LoRA for local paths (check adapter_config.json on disk)
         if not is_lora and is_local:
             detected_base = get_base_model_from_lora(path)
diff --git a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
index 3dca94d7f5..030fe55bff 100644
--- a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
+++ b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
@@ -41,11 +41,13 @@ function stripTrailingEpoch(input: string): string {
 function describeModel(model: {
   is_lora?: boolean;
   is_vision?: boolean;
+  is_gguf?: boolean;
 }): string | undefined {
   const tags: string[] = [];
+  if (model.is_gguf) tags.push("GGUF");
   if (model.is_lora) tags.push("LoRA");
   if (model.is_vision) tags.push("Vision");
-  if (!model.is_lora && !model.is_vision) tags.push("Base");
+  if (!model.is_lora && !model.is_vision && !model.is_gguf) tags.push("Base");
   return tags.join(" · ");
 }
 
@@ -54,6 +56,7 @@ function toChatModelSummary(model: {
   name?: string | null;
   is_lora?: boolean;
   is_vision?: boolean;
+  is_gguf?: boolean;
 }): ChatModelSummary {
   return {
     id: model.id,
@@ -61,6 +64,7 @@ function toChatModelSummary(model: {
     description: describeModel(model),
     isLora: Boolean(model.is_lora),
     isVision: Boolean(model.is_vision),
+    isGguf: Boolean(model.is_gguf),
   };
 }
 
diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.ts
index cadcad152d..bc9268e2c2 100644
--- a/studio/frontend/src/features/chat/types/api.ts
+++ b/studio/frontend/src/features/chat/types/api.ts
@@ -3,6 +3,7 @@ export interface BackendModelDetails {
   name?: string | null;
   is_vision?: boolean;
   is_lora?: boolean;
+  is_gguf?: boolean;
 }
 
 export interface ListModelsResponse {
@@ -35,6 +36,7 @@ export interface LoadModelResponse {
   display_name: string;
   is_vision: boolean;
   is_lora: boolean;
+  is_gguf?: boolean;
   inference?: {
     temperature?: number;
     top_p?: number;
@@ -50,6 +52,7 @@ export interface UnloadModelRequest {
 export interface InferenceStatusResponse {
   active_model: string | null;
   is_vision: boolean;
+  is_gguf?: boolean;
   loading: string[];
   loaded: string[];
 }
diff --git a/studio/frontend/src/features/chat/types/runtime.ts b/studio/frontend/src/features/chat/types/runtime.ts
index 47478e5aae..9f392fe08d 100644
--- a/studio/frontend/src/features/chat/types/runtime.ts
+++ b/studio/frontend/src/features/chat/types/runtime.ts
@@ -26,6 +26,7 @@ export interface ChatModelSummary {
   description?: string;
   isVision: boolean;
   isLora: boolean;
+  isGguf?: boolean;
 }
 
 export interface ChatLoraSummary {
diff --git a/studio/frontend/src/hooks/use-hf-model-search.ts b/studio/frontend/src/hooks/use-hf-model-search.ts
index 6ba70a4d5c..0029f13317 100644
--- a/studio/frontend/src/hooks/use-hf-model-search.ts
+++ b/studio/frontend/src/hooks/use-hf-model-search.ts
@@ -11,7 +11,6 @@ export interface HfModelResult {
 }
 
 const EXCLUDED_TAGS = new Set([
-  "gguf",
   "gptq",
   "awq",
   "exl2",

From eb6e9f7412d1b28678c6005bdd11a777d8e67759 Mon Sep 17 00:00:00 2001
From: Roland Tannous <115670425+rolandtannous@users.noreply.github.com>
Date: Tue, 24 Feb 2026 17:45:17 +0400
Subject: [PATCH 02/12] Fix CUDA detection for llama-server build on multi-GPU
 machines

---
 setup.sh | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/setup.sh b/setup.sh
index be17370df4..68c2d010da 100755
--- a/setup.sh
+++ b/setup.sh
@@ -233,9 +233,25 @@ else
 
         if [ "$BUILD_OK" = true ]; then
             CMAKE_ARGS=""
+            # Detect CUDA: check nvcc on PATH, then common install locations
+            NVCC_PATH=""
             if command -v nvcc &>/dev/null; then
-                echo "   Building with CUDA support..."
+                NVCC_PATH="$(command -v nvcc)"
+            elif [ -x /usr/local/cuda/bin/nvcc ]; then
+                NVCC_PATH="/usr/local/cuda/bin/nvcc"
+                export PATH="/usr/local/cuda/bin:$PATH"
+            elif ls /usr/local/cuda-*/bin/nvcc &>/dev/null 2>&1; then
+                # Pick the newest cuda-XX.X directory
+                NVCC_PATH="$(ls -d /usr/local/cuda-*/bin/nvcc 2>/dev/null | sort -V | tail -1)"
+                export PATH="$(dirname "$NVCC_PATH"):$PATH"
+            fi
+
+            if [ -n "$NVCC_PATH" ]; then
+                echo "   Building with CUDA support (nvcc: $NVCC_PATH)..."
                 CMAKE_ARGS="-DGGML_CUDA=ON"
+            elif [ -d /usr/local/cuda ] || nvidia-smi &>/dev/null; then
+                echo "   CUDA driver detected but nvcc not found — building CPU-only"
+                echo "   To enable GPU: install cuda-toolkit or add nvcc to PATH"
             else
                 echo "   Building CPU-only (no CUDA detected)..."
             fi

From c635d4f49cddc09ad8aa5b2c01e354529c2282f3 Mon Sep 17 00:00:00 2001
From: Roland Tannous <115670425+rolandtannous@users.noreply.github.com>
Date: Tue, 24 Feb 2026 17:49:09 +0400
Subject: [PATCH 03/12] Fix GGUF detection for HuggingFace repo IDs (not just
 local paths)

---
 studio/backend/utils/models/model_config.py | 101 +++++++++++++++++++-
 1 file changed, 97 insertions(+), 4 deletions(-)

diff --git a/studio/backend/utils/models/model_config.py b/studio/backend/utils/models/model_config.py
index 27fbe3ee44..53ac28be65 100644
--- a/studio/backend/utils/models/model_config.py
+++ b/studio/backend/utils/models/model_config.py
@@ -424,14 +424,14 @@ pass
 
 def detect_gguf_model(path: str) -> Optional[str]:
     """
-    Check if the given path is or contains a GGUF model file.
+    Check if the given local path is or contains a GGUF model file.
 
-    Handles three cases:
+    Handles two cases:
     1. path is a direct .gguf file path
     2. path is a directory containing .gguf files
-    3. path is an HuggingFace repo with GGUF files (not yet — future enhancement)
 
     Returns the full path to the .gguf file if found, None otherwise.
+    For HuggingFace repo detection, use detect_gguf_model_remote() instead.
     """
     p = Path(path)
 
@@ -448,6 +448,76 @@ def detect_gguf_model(path: str) -> Optional[str]:
     return None
 
 
+# Preferred GGUF quantization levels, in descending priority.
+# Q4_K_M is a good default: small, fast, acceptable quality.
+_GGUF_QUANT_PREFERENCE = [
+    "Q4_K_M", "Q4_K_S", "Q5_K_M", "Q5_K_S",
+    "Q6_K", "Q8_0", "Q3_K_M", "Q3_K_L", "Q2_K",
+    "F16", "BF16", "F32",
+]
+
+
+def _pick_best_gguf(filenames: list[str]) -> Optional[str]:
+    """
+    Pick the best GGUF file from a list of filenames.
+
+    Prefers quantization levels in _GGUF_QUANT_PREFERENCE order.
+    Falls back to the first .gguf file found.
+    """
+    gguf_files = [f for f in filenames if f.endswith(".gguf")]
+    if not gguf_files:
+        return None
+
+    # Try preferred quantization levels
+    for quant in _GGUF_QUANT_PREFERENCE:
+        for f in gguf_files:
+            if quant in f:
+                return f
+
+    # Fallback: first GGUF file
+    return gguf_files[0]
+
+
+def detect_gguf_model_remote(
+    repo_id: str,
+    hf_token: Optional[str] = None,
+) -> Optional[str]:
+    """
+    Check if a HuggingFace repo contains GGUF files.
+
+    Returns the filename of the best GGUF file in the repo, or None.
+    """
+    try:
+        from huggingface_hub import model_info as hf_model_info
+
+        info = hf_model_info(repo_id, token=hf_token)
+        repo_files = [s.rfilename for s in info.siblings]
+        return _pick_best_gguf(repo_files)
+    except Exception as e:
+        logger.debug(f"Could not check GGUF files for '{repo_id}': {e}")
+        return None
+
+
+def download_gguf_file(
+    repo_id: str,
+    filename: str,
+    hf_token: Optional[str] = None,
+) -> str:
+    """
+    Download a specific GGUF file from a HuggingFace repo.
+
+    Returns the local path to the downloaded file.
+    """
+    from huggingface_hub import hf_hub_download
+
+    local_path = hf_hub_download(
+        repo_id=repo_id,
+        filename=filename,
+        token=hf_token,
+    )
+    return local_path
+
+
 def scan_trained_loras(outputs_dir: str = "./outputs") -> List[Tuple[str, str]]:
     """
     Scan outputs folder for trained LoRA adapters.
@@ -714,7 +784,7 @@ class ModelConfig:
             gguf_file = detect_gguf_model(path)
             if gguf_file:
                 display_name = Path(gguf_file).stem
-                logger.info(f"Detected GGUF model: {gguf_file}")
+                logger.info(f"Detected local GGUF model: {gguf_file}")
                 return cls(
                     identifier=identifier,
                     display_name=display_name,
@@ -726,6 +796,29 @@ class ModelConfig:
                     is_gguf=True,
                     gguf_file=gguf_file,
                 )
+        else:
+            # Check if the HF repo contains GGUF files
+            gguf_filename = detect_gguf_model_remote(identifier, hf_token=hf_token)
+            if gguf_filename:
+                logger.info(f"Detected remote GGUF repo '{identifier}', file: {gguf_filename}")
+                logger.info(f"Downloading GGUF file '{gguf_filename}' from '{identifier}'...")
+                local_gguf_path = download_gguf_file(
+                    repo_id=identifier,
+                    filename=gguf_filename,
+                    hf_token=hf_token,
+                )
+                display_name = Path(gguf_filename).stem
+                return cls(
+                    identifier=identifier,
+                    display_name=display_name,
+                    path=local_gguf_path,
+                    is_local=False,
+                    is_cached=True,
+                    is_vision=False,
+                    is_lora=False,
+                    is_gguf=True,
+                    gguf_file=local_gguf_path,
+                )
 
         # Auto-detect LoRA for local paths (check adapter_config.json on disk)
         if not is_lora and is_local:

From 4a82e704aa93f28b4e2c45a32442409d6379d838 Mon Sep 17 00:00:00 2001
From: Roland Tannous <115670425+rolandtannous@users.noreply.github.com>
Date: Tue, 24 Feb 2026 18:02:43 +0400
Subject: [PATCH 04/12] Preflight llama-server check before downloading remote
 GGUF files

---
 studio/backend/utils/models/model_config.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/studio/backend/utils/models/model_config.py b/studio/backend/utils/models/model_config.py
index 53ac28be65..601ea75770 100644
--- a/studio/backend/utils/models/model_config.py
+++ b/studio/backend/utils/models/model_config.py
@@ -800,6 +800,15 @@ class ModelConfig:
             # Check if the HF repo contains GGUF files
             gguf_filename = detect_gguf_model_remote(identifier, hf_token=hf_token)
             if gguf_filename:
+                # Preflight: verify llama-server binary exists before downloading
+                # a potentially multi-GB GGUF file
+                from core.inference.llama_cpp import LlamaCppBackend
+                if not LlamaCppBackend._find_llama_server_binary():
+                    raise RuntimeError(
+                        "llama-server binary not found — cannot load GGUF models. "
+                        "Run setup.sh to build it, or set LLAMA_SERVER_PATH."
+                    )
+
                 logger.info(f"Detected remote GGUF repo '{identifier}', file: {gguf_filename}")
                 logger.info(f"Downloading GGUF file '{gguf_filename}' from '{identifier}'...")
                 local_gguf_path = download_gguf_file(

From 5b7555cd3f843578487a849b91c5cf8c94569b8c Mon Sep 17 00:00:00 2001
From: Roland Tannous <115670425+rolandtannous@users.noreply.github.com>
Date: Tue, 24 Feb 2026 18:19:29 +0400
Subject: [PATCH 05/12] Fix llama-server: build in-tree, fix path resolution,
 add LD_LIBRARY_PATH

---
 .gitignore                                 |  3 ++
 setup.sh                                   | 32 +++++++++++----------
 studio/backend/core/inference/llama_cpp.py | 33 ++++++++++++++--------
 3 files changed, 42 insertions(+), 26 deletions(-)

diff --git a/.gitignore b/.gitignore
index e24c38c2b0..2ede66ec5b 100755
--- a/.gitignore
+++ b/.gitignore
@@ -24,6 +24,9 @@ unsloth_training_checkpoints/
 *.gguf
 *.safetensors
 
+# llama.cpp build (built by setup.sh, shared with unsloth-zoo export)
+llama.cpp/
+
 # Built binaries (llama-server etc.)
 bin/
 
diff --git a/setup.sh b/setup.sh
index 68c2d010da..36165dac65 100755
--- a/setup.sh
+++ b/setup.sh
@@ -207,10 +207,10 @@ else
 fi
 
 # ── 8. Build llama-server for GGUF inference ──
-# Builds in an isolated temp directory to avoid conflicts with unsloth-zoo's
-# own llama.cpp management (used for GGUF export). Only the llama-server
-# binary is extracted to $REPO/bin/.
-LLAMA_SERVER_BIN="$SCRIPT_DIR/bin/llama-server"
+# Builds in-tree at $REPO/llama.cpp/. This directory is shared with
+# unsloth-zoo's GGUF export pipeline — if converter/quantize are missing,
+# unsloth-zoo will rebuild them on first export. We only build llama-server here.
+LLAMA_SERVER_BIN="$SCRIPT_DIR/llama.cpp/build/bin/llama-server"
 if [ -f "$LLAMA_SERVER_BIN" ]; then
     echo ""
     echo "✅ llama-server already exists at $LLAMA_SERVER_BIN"
@@ -226,10 +226,17 @@ else
     else
         echo ""
         echo "Building llama-server for GGUF inference..."
-        LLAMA_BUILD_TMP=$(mktemp -d)
+        LLAMA_CPP_DIR="$SCRIPT_DIR/llama.cpp"
 
         BUILD_OK=true
-        run_quiet "clone llama.cpp" git clone --depth 1 https://github.com/ggml-org/llama.cpp.git "$LLAMA_BUILD_TMP/llama.cpp" || BUILD_OK=false
+        if [ -d "$LLAMA_CPP_DIR/.git" ]; then
+            echo "   llama.cpp repo already cloned, pulling latest..."
+            run_quiet "pull llama.cpp" git -C "$LLAMA_CPP_DIR" pull || true
+        else
+            # Remove any non-git llama.cpp directory (stale build artifacts)
+            rm -rf "$LLAMA_CPP_DIR"
+            run_quiet "clone llama.cpp" git clone --depth 1 https://github.com/ggml-org/llama.cpp.git "$LLAMA_CPP_DIR" || BUILD_OK=false
+        fi
 
         if [ "$BUILD_OK" = true ]; then
             CMAKE_ARGS=""
@@ -258,27 +265,22 @@ else
 
             NCPU=$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4)
 
-            run_quiet "cmake llama.cpp" cmake -S "$LLAMA_BUILD_TMP/llama.cpp" -B "$LLAMA_BUILD_TMP/llama.cpp/build" $CMAKE_ARGS || BUILD_OK=false
+            run_quiet "cmake llama.cpp" cmake -S "$LLAMA_CPP_DIR" -B "$LLAMA_CPP_DIR/build" $CMAKE_ARGS || BUILD_OK=false
         fi
 
         if [ "$BUILD_OK" = true ]; then
-            run_quiet "build llama-server" cmake --build "$LLAMA_BUILD_TMP/llama.cpp/build" --config Release --target llama-server -j"$NCPU" || BUILD_OK=false
+            run_quiet "build llama-server" cmake --build "$LLAMA_CPP_DIR/build" --config Release --target llama-server -j"$NCPU" || BUILD_OK=false
         fi
 
         if [ "$BUILD_OK" = true ]; then
-            mkdir -p "$SCRIPT_DIR/bin"
-            if [ -f "$LLAMA_BUILD_TMP/llama.cpp/build/bin/llama-server" ]; then
-                cp "$LLAMA_BUILD_TMP/llama.cpp/build/bin/llama-server" "$LLAMA_SERVER_BIN"
-                echo "✅ llama-server built and installed to $LLAMA_SERVER_BIN"
+            if [ -f "$LLAMA_SERVER_BIN" ]; then
+                echo "✅ llama-server built at $LLAMA_SERVER_BIN"
             else
                 echo "⚠️  llama-server binary not found after build — GGUF inference won't be available"
             fi
         else
             echo "⚠️  llama-server build failed — GGUF inference won't be available, but everything else works"
         fi
-
-        # Clean up temp build directory
-        rm -rf "$LLAMA_BUILD_TMP"
     fi
 fi
 
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index dae8d20489..1489365686 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -65,9 +65,9 @@ class LlamaCppBackend:
 
         Search order:
         1. LLAMA_SERVER_PATH environment variable
-        2. ./bin/llama-server  (built by setup.sh)
+        2. ./llama.cpp/build/bin/llama-server  (built by setup.sh in-tree)
         3. llama-server on PATH  (system install)
-        4. ./llama.cpp/llama-server  (unsloth-zoo build output)
+        4. ./bin/llama-server  (legacy: extracted binary)
         """
         import os
 
@@ -76,21 +76,23 @@ class LlamaCppBackend:
         if env_path and Path(env_path).is_file():
             return env_path
 
-        # 2. Project bin/ directory (setup.sh output)
-        project_root = Path(__file__).resolve().parents[3]  # core/inference/ → backend/ → studio/ → root
-        bin_path = project_root / "bin" / "llama-server"
-        if bin_path.is_file():
-            return str(bin_path)
+        # Project root: llama_cpp.py → inference/ → core/ → backend/ → studio/ → root
+        project_root = Path(__file__).resolve().parents[4]
+
+        # 2. In-tree llama.cpp build (setup.sh builds here)
+        build_path = project_root / "llama.cpp" / "build" / "bin" / "llama-server"
+        if build_path.is_file():
+            return str(build_path)
 
         # 3. System PATH
         system_path = shutil.which("llama-server")
         if system_path:
             return system_path
 
-        # 4. unsloth-zoo build output (from GGUF export)
-        llama_cpp_path = project_root / "llama.cpp" / "llama-server"
-        if llama_cpp_path.is_file():
-            return str(llama_cpp_path)
+        # 4. Legacy: extracted to bin/
+        bin_path = project_root / "bin" / "llama-server"
+        if bin_path.is_file():
+            return str(bin_path)
 
         return None
 
@@ -154,11 +156,20 @@ class LlamaCppBackend:
 
             logger.info(f"Starting llama-server: {' '.join(cmd)}")
 
+            # Set LD_LIBRARY_PATH so llama-server can find its shared libs
+            # (libmtmd.so, libllama.so, etc.) which live next to the binary
+            import os
+            env = os.environ.copy()
+            binary_dir = str(Path(binary).parent)
+            existing_ld = env.get("LD_LIBRARY_PATH", "")
+            env["LD_LIBRARY_PATH"] = f"{binary_dir}:{existing_ld}" if existing_ld else binary_dir
+
             self._process = subprocess.Popen(
                 cmd,
                 stdout=subprocess.PIPE,
                 stderr=subprocess.STDOUT,
                 text=True,
+                env=env,
             )
 
             self._gguf_path = gguf_path

From 3ee4f1359a22c6b3abb2b12a3d44bc3c06e36208 Mon Sep 17 00:00:00 2001
From: Roland Tannous <115670425+rolandtannous@users.noreply.github.com>
Date: Tue, 24 Feb 2026 19:03:06 +0400
Subject: [PATCH 06/12] Use llama-server -hf mode, add GGUF variant selector,
 fix vision detection

Replace Python-side GGUF download with llama-server's native -hf flag for
HuggingFace repos. Add frontend variant picker so users can choose
quantization (Q4_K_M, Q8_0, BF16, etc.) with file sizes. Fix vision
detection via mmproj files instead of hardcoding is_vision=False.
---
 studio/backend/core/inference/llama_cpp.py    |  90 +++++---
 studio/backend/models/inference.py            |   1 +
 studio/backend/models/models.py               |  15 ++
 studio/backend/routes/inference.py            |  35 ++-
 studio/backend/routes/models.py               |  48 ++++
 studio/backend/utils/models/__init__.py       |   4 +
 studio/backend/utils/models/model_config.py   | 132 +++++++++--
 .../assistant-ui/model-selector/pickers.tsx   | 208 +++++++++++++++---
 .../assistant-ui/model-selector/types.ts      |   1 +
 .../src/features/chat/api/chat-api.ts         |  11 +
 .../frontend/src/features/chat/chat-page.tsx  |   8 +-
 .../chat/hooks/use-chat-model-runtime.ts      |   4 +
 .../frontend/src/features/chat/types/api.ts   |  14 ++
 13 files changed, 489 insertions(+), 82 deletions(-)

diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 1489365686..b5c7d3c50d 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -36,6 +36,9 @@ class LlamaCppBackend:
         self._port: Optional[int] = None
         self._model_identifier: Optional[str] = None
         self._gguf_path: Optional[str] = None
+        self._hf_repo: Optional[str] = None
+        self._hf_variant: Optional[str] = None
+        self._is_vision: bool = False
         self._healthy = False
         self._lock = threading.Lock()
         self._chat_template: Optional[str] = None
@@ -56,6 +59,10 @@ class LlamaCppBackend:
     def model_identifier(self) -> Optional[str]:
         return self._model_identifier
 
+    @property
+    def is_vision(self) -> bool:
+        return self._is_vision
+
     # ── Binary discovery ──────────────────────────────────────────
 
     @staticmethod
@@ -109,27 +116,33 @@ class LlamaCppBackend:
 
     def load_model(
         self,
-        gguf_path: str,
+        *,
+        # Local mode: pass a path to a .gguf file
+        gguf_path: Optional[str] = None,
+        # HF mode: let llama-server download via -hf "repo:quant"
+        hf_repo: Optional[str] = None,
+        hf_variant: Optional[str] = None,
+        hf_token: Optional[str] = None,
+        # Common
         model_identifier: str,
+        is_vision: bool = False,
         n_ctx: int = 4096,
         n_gpu_layers: int = -1,
         n_threads: Optional[int] = None,
     ) -> bool:
         """
-        Start llama-server with the given GGUF file.
+        Start llama-server with a GGUF model.
 
-        Args:
-            gguf_path: Path to the .gguf file
-            model_identifier: Display identifier for the model
-            n_ctx: Context window size
-            n_gpu_layers: Number of layers to offload to GPU (-1 = all)
-            n_threads: Number of CPU threads (None = auto)
+        Two modes:
+        - Local: ``gguf_path="/path/to/model.gguf"`` → uses ``-m``
+        - HF:    ``hf_repo="unsloth/gemma-3-4b-it-GGUF", hf_variant="Q4_K_M"`` → uses ``-hf``
 
-        Returns:
-            True if server started and health check passed.
+        In HF mode, llama-server handles downloading, caching, and
+        auto-loading mmproj files for vision models.
+
+        Returns True if server started and health check passed.
         """
         with self._lock:
-            # Kill existing process if any
             self._kill_process()
 
             binary = self._find_llama_server_binary()
@@ -140,17 +153,33 @@ class LlamaCppBackend:
                     "or set LLAMA_SERVER_PATH environment variable."
                 )
 
-            if not Path(gguf_path).is_file():
-                raise FileNotFoundError(f"GGUF file not found: {gguf_path}")
-
             self._port = self._find_free_port()
-            cmd = [
-                binary,
-                "-m", gguf_path,
-                "--port", str(self._port),
-                "-c", str(n_ctx),
-                "-ngl", str(n_gpu_layers),
-            ]
+
+            # Build command based on mode
+            if hf_repo:
+                hf_spec = f"{hf_repo}:{hf_variant}" if hf_variant else hf_repo
+                cmd = [
+                    binary,
+                    "-hf", hf_spec,
+                    "--port", str(self._port),
+                    "-c", str(n_ctx),
+                    "-ngl", str(n_gpu_layers),
+                ]
+                if hf_token:
+                    cmd.extend(["--hf-token", hf_token])
+            elif gguf_path:
+                if not Path(gguf_path).is_file():
+                    raise FileNotFoundError(f"GGUF file not found: {gguf_path}")
+                cmd = [
+                    binary,
+                    "-m", gguf_path,
+                    "--port", str(self._port),
+                    "-c", str(n_ctx),
+                    "-ngl", str(n_gpu_layers),
+                ]
+            else:
+                raise ValueError("Either gguf_path or hf_repo must be provided")
+
             if n_threads is not None:
                 cmd.extend(["--threads", str(n_threads)])
 
@@ -173,10 +202,14 @@ class LlamaCppBackend:
             )
 
             self._gguf_path = gguf_path
+            self._hf_repo = hf_repo
+            self._hf_variant = hf_variant
+            self._is_vision = is_vision
             self._model_identifier = model_identifier
 
-            # Wait for health
-            if not self._wait_for_health(timeout=120.0):
+            # HF mode: llama-server downloads before becoming healthy — need longer timeout
+            timeout = 600.0 if hf_repo else 120.0
+            if not self._wait_for_health(timeout=timeout):
                 self._kill_process()
                 raise RuntimeError(
                     "llama-server failed to start. "
@@ -185,8 +218,12 @@ class LlamaCppBackend:
 
             self._healthy = True
 
-            # Try to read chat template from GGUF metadata
-            self._chat_template = self._read_gguf_chat_template(gguf_path)
+            # Read chat template from local GGUF metadata (skip in HF mode —
+            # llama-server handles template application internally)
+            if gguf_path:
+                self._chat_template = self._read_gguf_chat_template(gguf_path)
+            else:
+                self._chat_template = None
 
             logger.info(
                 f"llama-server ready on port {self._port} "
@@ -201,6 +238,9 @@ class LlamaCppBackend:
             logger.info(f"Unloaded GGUF model: {self._model_identifier}")
             self._model_identifier = None
             self._gguf_path = None
+            self._hf_repo = None
+            self._hf_variant = None
+            self._is_vision = False
             self._port = None
             self._healthy = False
             self._chat_template = None
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py
index c4a062ae3a..fc3f788ac6 100644
--- a/studio/backend/models/inference.py
+++ b/studio/backend/models/inference.py
@@ -17,6 +17,7 @@ class LoadRequest(BaseModel):
     max_seq_length: int = Field(2048, ge=128, le=32768, description="Maximum sequence length")
     load_in_4bit: bool = Field(True, description="Load model in 4-bit quantization")
     is_lora: bool = Field(False, description="Whether this is a LoRA adapter")
+    gguf_variant: Optional[str] = Field(None, description="GGUF quantization variant (e.g. 'Q4_K_M')")
 
 
 class UnloadRequest(BaseModel):
diff --git a/studio/backend/models/models.py b/studio/backend/models/models.py
index b3f9b50ed2..bd035fcbee 100644
--- a/studio/backend/models/models.py
+++ b/studio/backend/models/models.py
@@ -76,6 +76,21 @@ class ModelListResponse(BaseModel):
     default_models: List[str] = Field(default_factory=list, description="List of default model IDs")
 
 
+class GgufVariantDetail(BaseModel):
+    """A single GGUF quantization variant in a HuggingFace repo."""
+    filename: str = Field(..., description="GGUF filename (e.g., 'gemma-3-4b-it-Q4_K_M.gguf')")
+    quant: str = Field(..., description="Quantization label (e.g., 'Q4_K_M')")
+    size_bytes: int = Field(0, description="File size in bytes")
+
+
+class GgufVariantsResponse(BaseModel):
+    """Response for listing GGUF quantization variants in a HuggingFace repo."""
+    repo_id: str = Field(..., description="HuggingFace repo ID")
+    variants: List[GgufVariantDetail] = Field(default_factory=list, description="Available GGUF variants")
+    has_vision: bool = Field(False, description="Whether the model has vision support (mmproj files)")
+    default_variant: Optional[str] = Field(None, description="Recommended default quantization variant")
+
+
 class LocalModelInfo(BaseModel):
     """Discovered local model candidate."""
     id: str = Field(..., description="Identifier to use for loading/training")
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 156578b960..b7a7a33b43 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -87,6 +87,7 @@ async def load_model(request: LoadRequest):
         config = ModelConfig.from_identifier(
             model_id=request.model_path,
             hf_token=request.hf_token,
+            gguf_variant=request.gguf_variant,
         )
 
         if not config:
@@ -105,11 +106,25 @@ async def load_model(request: LoadRequest):
                 logger.info(f"Unloading Unsloth model '{unsloth_backend.active_model_name}' before loading GGUF")
                 unsloth_backend.unload_model(unsloth_backend.active_model_name)
 
-            success = llama_backend.load_model(
-                gguf_path=config.gguf_file,
-                model_identifier=config.identifier,
-                n_ctx=request.max_seq_length,
-            )
+            # Route to HF mode or local mode based on config
+            if config.gguf_hf_repo:
+                # HF mode: llama-server downloads via -hf "repo:quant"
+                success = llama_backend.load_model(
+                    hf_repo=config.gguf_hf_repo,
+                    hf_variant=config.gguf_variant,
+                    hf_token=request.hf_token,
+                    model_identifier=config.identifier,
+                    is_vision=config.is_vision,
+                    n_ctx=request.max_seq_length,
+                )
+            else:
+                # Local mode: llama-server loads via -m <path>
+                success = llama_backend.load_model(
+                    gguf_path=config.gguf_file,
+                    model_identifier=config.identifier,
+                    is_vision=config.is_vision,
+                    n_ctx=request.max_seq_length,
+                )
 
             if not success:
                 raise HTTPException(
@@ -125,7 +140,7 @@ async def load_model(request: LoadRequest):
                 status="loaded",
                 model=config.identifier,
                 display_name=config.display_name,
-                is_vision=False,
+                is_vision=config.is_vision,
                 is_lora=False,
                 is_gguf=True,
                 inference=inference_config,
@@ -292,7 +307,7 @@ async def get_status():
         if llama_backend.is_loaded:
             return InferenceStatusResponse(
                 active_model=llama_backend.model_identifier,
-                is_vision=False,
+                is_vision=llama_backend.is_vision,
                 is_gguf=True,
                 loading=[],
                 loaded=[llama_backend.model_identifier],
@@ -425,12 +440,12 @@ async def openai_chat_completions(payload: ChatCompletionRequest, request: Reque
 
     # ── GGUF path: format prompt → proxy to llama-server ──────
     if using_gguf:
-        # GGUF models don't support vision
+        # Reject images if this GGUF model doesn't support vision
         image_b64 = extracted_image_b64 or payload.image_base64
-        if image_b64:
+        if image_b64 and not llama_backend.is_vision:
             raise HTTPException(
                 status_code=400,
-                detail="Image provided but GGUF models do not support vision.",
+                detail="Image provided but current GGUF model does not support vision.",
             )
 
         prompt = llama_backend.format_prompt(chat_messages, system_prompt)
diff --git a/studio/backend/routes/models.py b/studio/backend/routes/models.py
index 881f399c3d..31cc36cc45 100644
--- a/studio/backend/routes/models.py
+++ b/studio/backend/routes/models.py
@@ -22,8 +22,10 @@ try:
         get_base_model_from_lora,
         is_vision_model,
         scan_checkpoints,
+        list_gguf_variants,
         ModelConfig,
     )
+    from utils.models.model_config import _pick_best_gguf, _extract_quant_label
     from core.inference import get_inference_backend
 except ImportError:
     # Fallback: try to import from parent directory
@@ -36,8 +38,10 @@ except ImportError:
         get_base_model_from_lora,
         is_vision_model,
         scan_checkpoints,
+        list_gguf_variants,
         ModelConfig,
     )
+    from utils.models.model_config import _pick_best_gguf, _extract_quant_label
     from core.inference import get_inference_backend
 
 from models import (
@@ -51,6 +55,7 @@ from models import (
     LoRAInfo,
     ModelListResponse,
 )
+from models.models import GgufVariantDetail, GgufVariantsResponse
 from models.responses import LoRABaseModelResponse, VisionCheckResponse
 
 router = APIRouter()
@@ -405,6 +410,49 @@ async def check_vision_model(
             detail=f"Failed to check vision model: {str(e)}"
         )
 
+@router.get("/gguf-variants", response_model=GgufVariantsResponse)
+async def get_gguf_variants(
+    repo_id: str = Query(..., description="HuggingFace repo ID (e.g. 'unsloth/gemma-3-4b-it-GGUF')"),
+    hf_token: Optional[str] = Query(None, description="HuggingFace token for private repos"),
+    current_subject: str = Depends(get_current_subject),
+):
+    """
+    List available GGUF quantization variants for a HuggingFace repo.
+
+    Returns all available quantization variants (Q4_K_M, Q8_0, BF16, etc.)
+    with file sizes, whether the model supports vision, and the recommended
+    default variant.
+    """
+    try:
+        variants, has_vision = list_gguf_variants(repo_id, hf_token=hf_token)
+
+        # Determine default variant
+        filenames = [v.filename for v in variants]
+        best = _pick_best_gguf(filenames)
+        default_variant = _extract_quant_label(best) if best else None
+
+        return GgufVariantsResponse(
+            repo_id=repo_id,
+            variants=[
+                GgufVariantDetail(
+                    filename=v.filename,
+                    quant=v.quant,
+                    size_bytes=v.size_bytes,
+                )
+                for v in variants
+            ],
+            has_vision=has_vision,
+            default_variant=default_variant,
+        )
+
+    except Exception as e:
+        logger.error(f"Error listing GGUF variants for '{repo_id}': {e}", exc_info=True)
+        raise HTTPException(
+            status_code=500,
+            detail=f"Failed to list GGUF variants: {str(e)}",
+        )
+
+
 @router.get("/checkpoints", response_model=CheckpointListResponse)
 async def list_checkpoints(
     outputs_dir: str = Query(
diff --git a/studio/backend/utils/models/__init__.py b/studio/backend/utils/models/__init__.py
index 505fd35edd..4006e63908 100644
--- a/studio/backend/utils/models/__init__.py
+++ b/studio/backend/utils/models/__init__.py
@@ -3,11 +3,13 @@ Model and LoRA configuration handling
 """
 from .model_config import (
     ModelConfig,
+    GgufVariantInfo,
     is_vision_model,
     scan_trained_loras,
     load_model_defaults,
     get_base_model_from_lora,
     load_model_config,
+    list_gguf_variants,
     MODEL_NAME_MAPPING,
     UI_STATUS_INDICATORS,
 )
@@ -15,11 +17,13 @@ from .checkpoints import scan_checkpoints
 
 __all__ = [
     'ModelConfig',
+    'GgufVariantInfo',
     'is_vision_model',
     'scan_trained_loras',
     'load_model_defaults',
     'get_base_model_from_lora',
     'load_model_config',
+    'list_gguf_variants',
     'MODEL_NAME_MAPPING',
     'UI_STATUS_INDICATORS',
     'scan_checkpoints',
diff --git a/studio/backend/utils/models/model_config.py b/studio/backend/utils/models/model_config.py
index 601ea75770..0c29d1c869 100644
--- a/studio/backend/utils/models/model_config.py
+++ b/studio/backend/utils/models/model_config.py
@@ -478,6 +478,83 @@ def _pick_best_gguf(filenames: list[str]) -> Optional[str]:
     return gguf_files[0]
 
 
+@dataclass
+class GgufVariantInfo:
+    """A single GGUF quantization variant from a HuggingFace repo."""
+    filename: str       # e.g., "gemma-3-4b-it-Q4_K_M.gguf"
+    quant: str          # e.g., "Q4_K_M" (extracted from filename)
+    size_bytes: int     # file size
+
+
+def _extract_quant_label(filename: str) -> str:
+    """
+    Extract quantization label like Q4_K_M, IQ4_XS, BF16 from a GGUF filename.
+
+    Examples:
+        "gemma-3-4b-it-Q4_K_M.gguf" → "Q4_K_M"
+        "model-IQ4_NL.gguf"          → "IQ4_NL"
+        "model-BF16.gguf"            → "BF16"
+        "model-UD-IQ1_S.gguf"        → "UD-IQ1_S"
+    """
+    import re
+    stem = filename.rsplit(".", 1)[0]  # Remove .gguf
+    # Match known quantization patterns (UD- prefix, IQ, Q, BF/F variants)
+    match = re.search(
+        r'(UD-)?'  # Optional UD- prefix (Ultra Discrete)
+        r'(IQ[0-9]+_[A-Z]+(?:_[A-Z0-9]+)?'  # IQ variants: IQ4_XS, IQ4_NL, IQ1_S
+        r'|Q[0-9]+_K_[A-Z]+'                  # K-quant: Q4_K_M, Q3_K_S
+        r'|Q[0-9]+_[0-9]+'                    # Standard: Q8_0, Q5_1
+        r'|Q[0-9]+_K'                          # Short K-quant: Q6_K
+        r'|BF16|F16|F32)',                     # Full precision
+        stem, re.IGNORECASE,
+    )
+    if match:
+        prefix = match.group(1) or ""
+        return f"{prefix}{match.group(2)}"
+    # Fallback: last segment after hyphen
+    return stem.split("-")[-1]
+
+
+def list_gguf_variants(
+    repo_id: str,
+    hf_token: Optional[str] = None,
+) -> tuple[list[GgufVariantInfo], bool]:
+    """
+    List all GGUF quantization variants in a HuggingFace repo.
+
+    Separates main model files from mmproj (vision projection) files.
+    The presence of mmproj files indicates a vision-capable model.
+
+    Returns:
+        (variants, has_vision): list of non-mmproj GGUF variants + vision flag.
+    """
+    from huggingface_hub import model_info as hf_model_info
+
+    info = hf_model_info(repo_id, token=hf_token)
+    variants: list[GgufVariantInfo] = []
+    has_vision = False
+
+    for sibling in info.siblings:
+        fname = sibling.rfilename
+        if not fname.endswith(".gguf"):
+            continue
+        size = sibling.size or 0
+
+        # mmproj files are vision projection models, not main model files
+        if "mmproj" in fname.lower():
+            has_vision = True
+            continue
+
+        quant = _extract_quant_label(fname)
+        variants.append(GgufVariantInfo(
+            filename=fname,
+            quant=quant,
+            size_bytes=size,
+        ))
+
+    return variants, has_vision
+
+
 def detect_gguf_model_remote(
     repo_id: str,
     hf_token: Optional[str] = None,
@@ -692,7 +769,9 @@ class ModelConfig:
     is_vision: bool     # Is this a vision model?
     is_lora: bool       # Is this a lora adapter?
     is_gguf: bool = False       # Is this a GGUF model?
-    gguf_file: Optional[str] = None  # Full path to the .gguf file
+    gguf_file: Optional[str] = None  # Full path to the .gguf file (local mode)
+    gguf_hf_repo: Optional[str] = None  # HF repo ID for -hf mode (e.g. "unsloth/gemma-3-4b-it-GGUF")
+    gguf_variant: Optional[str] = None  # Quantization variant (e.g. "Q4_K_M")
     base_model: Optional[str] = None  # Base model (for LoRAs)
 
     @classmethod
@@ -748,28 +827,32 @@ class ModelConfig:
         cls,
         model_id: str,
         hf_token: Optional[str] = None,
-        is_lora: bool = False
+        is_lora: bool = False,
+        gguf_variant: Optional[str] = None,
     ) -> Optional['ModelConfig']:
         """
         Create ModelConfig from a clean model identifier.
-        
+
         For FastAPI routes where the frontend sends sanitized model paths.
         No Gradio dropdown parsing - expects clean identifiers like:
         - "unsloth/Meta-Llama-3.1-8B-Instruct-bnb-4bit"
         - "./outputs/my_lora_adapter"
         - "/absolute/path/to/model"
-        
+
         Args:
             model_id: Clean model identifier (HF repo name or local path)
             hf_token: Optional HF token for vision detection on gated models
             is_lora: Whether this is a LoRA adapter
-        
+            gguf_variant: Optional GGUF quantization variant (e.g. "Q4_K_M").
+                For remote GGUF repos, specifies which quant to load via -hf.
+                If None, auto-selects using _pick_best_gguf().
+
         Returns:
             ModelConfig or None if configuration cannot be created
         """
         if not model_id or not model_id.strip():
             return None
-        
+
         identifier = model_id.strip()
         is_local = is_local_path(identifier)
         path = normalize_path(identifier) if is_local else identifier
@@ -800,8 +883,8 @@ class ModelConfig:
             # Check if the HF repo contains GGUF files
             gguf_filename = detect_gguf_model_remote(identifier, hf_token=hf_token)
             if gguf_filename:
-                # Preflight: verify llama-server binary exists before downloading
-                # a potentially multi-GB GGUF file
+                # Preflight: verify llama-server binary exists BEFORE user waits
+                # for a multi-GB download that llama-server handles natively
                 from core.inference.llama_cpp import LlamaCppBackend
                 if not LlamaCppBackend._find_llama_server_binary():
                     raise RuntimeError(
@@ -809,24 +892,35 @@ class ModelConfig:
                         "Run setup.sh to build it, or set LLAMA_SERVER_PATH."
                     )
 
-                logger.info(f"Detected remote GGUF repo '{identifier}', file: {gguf_filename}")
-                logger.info(f"Downloading GGUF file '{gguf_filename}' from '{identifier}'...")
-                local_gguf_path = download_gguf_file(
-                    repo_id=identifier,
-                    filename=gguf_filename,
-                    hf_token=hf_token,
+                # Use list_gguf_variants() to detect vision & resolve variant
+                variants, has_vision = list_gguf_variants(identifier, hf_token=hf_token)
+                variant = gguf_variant
+                if not variant:
+                    # Auto-select best quantization
+                    variant_filenames = [v.filename for v in variants]
+                    best = _pick_best_gguf(variant_filenames)
+                    if best:
+                        variant = _extract_quant_label(best)
+                    else:
+                        variant = "Q4_K_M"  # Fallback — llama-server's own default
+
+                display_name = f"{identifier.split('/')[-1]} ({variant})"
+                logger.info(
+                    f"Detected remote GGUF repo '{identifier}', "
+                    f"variant={variant}, vision={has_vision}"
                 )
-                display_name = Path(gguf_filename).stem
                 return cls(
                     identifier=identifier,
                     display_name=display_name,
-                    path=local_gguf_path,
+                    path=identifier,
                     is_local=False,
-                    is_cached=True,
-                    is_vision=False,
+                    is_cached=False,
+                    is_vision=has_vision,
                     is_lora=False,
                     is_gguf=True,
-                    gguf_file=local_gguf_path,
+                    gguf_file=None,
+                    gguf_hf_repo=identifier,
+                    gguf_variant=variant,
                 )
 
         # Auto-detect LoRA for local paths (check adapter_config.json on disk)
diff --git a/studio/frontend/src/components/assistant-ui/model-selector/pickers.tsx b/studio/frontend/src/components/assistant-ui/model-selector/pickers.tsx
index 80e9dd0b6b..c61f33785b 100644
--- a/studio/frontend/src/components/assistant-ui/model-selector/pickers.tsx
+++ b/studio/frontend/src/components/assistant-ui/model-selector/pickers.tsx
@@ -5,6 +5,8 @@ import {
   TooltipContent,
   TooltipTrigger,
 } from "@/components/ui/tooltip";
+import { listGgufVariants } from "@/features/chat/api/chat-api";
+import type { GgufVariantDetail } from "@/features/chat/types/api";
 import {
   useDebouncedValue,
   useGpuInfo,
@@ -17,7 +19,7 @@ import type { VramFitStatus } from "@/lib/vram";
 import { checkVramFit, estimateLoadingVram } from "@/lib/vram";
 import { Search01Icon } from "@hugeicons/core-free-icons";
 import { HugeiconsIcon } from "@hugeicons/react";
-import { useMemo, useState, type ReactNode } from "react";
+import { useCallback, useEffect, useMemo, useState, type ReactNode } from "react";
 import type {
   LoraModelOption,
   ModelOption,
@@ -36,6 +38,15 @@ function ListLabel({ children }: { children: ReactNode }) {
   );
 }
 
+/** Format bytes to a human-readable size string. */
+function formatBytes(bytes: number): string {
+  if (bytes === 0) return "0 B";
+  const units = ["B", "KB", "MB", "GB", "TB"];
+  const i = Math.floor(Math.log(bytes) / Math.log(1024));
+  const value = bytes / 1024 ** i;
+  return `${value.toFixed(value < 10 ? 1 : 0)} ${units[i]}`;
+}
+
 function ModelRow({
   label,
   meta,
@@ -114,6 +125,124 @@ function ModelRow({
   return content;
 }
 
+// ── GGUF Variant Expander ────────────────────────────────────
+
+function GgufVariantExpander({
+  repoId,
+  onSelect,
+}: {
+  repoId: string;
+  onSelect: (id: string, meta: ModelSelectorChangeMeta) => void;
+}) {
+  const [variants, setVariants] = useState<GgufVariantDetail[] | null>(null);
+  const [defaultVariant, setDefaultVariant] = useState<string | null>(null);
+  const [hasVision, setHasVision] = useState(false);
+  const [loading, setLoading] = useState(true);
+  const [error, setError] = useState<string | null>(null);
+
+  useEffect(() => {
+    let canceled = false;
+    setLoading(true);
+    setError(null);
+
+    listGgufVariants(repoId)
+      .then((res) => {
+        if (canceled) return;
+        setVariants(res.variants);
+        setDefaultVariant(res.default_variant);
+        setHasVision(res.has_vision);
+      })
+      .catch((err) => {
+        if (canceled) return;
+        setError(err instanceof Error ? err.message : "Failed to load variants");
+      })
+      .finally(() => {
+        if (!canceled) setLoading(false);
+      });
+
+    return () => {
+      canceled = true;
+    };
+  }, [repoId]);
+
+  const handleVariantClick = useCallback(
+    (quant: string) => {
+      onSelect(repoId, {
+        source: "hub",
+        isLora: false,
+        ggufVariant: quant,
+      });
+    },
+    [repoId, onSelect],
+  );
+
+  if (loading) {
+    return (
+      <div className="flex items-center gap-2 px-5 py-2">
+        <Spinner className="size-3 text-muted-foreground" />
+        <span className="text-xs text-muted-foreground">Loading variants…</span>
+      </div>
+    );
+  }
+
+  if (error) {
+    return (
+      <div className="px-5 py-2 text-xs text-destructive">{error}</div>
+    );
+  }
+
+  if (!variants || variants.length === 0) {
+    return (
+      <div className="px-5 py-2 text-xs text-muted-foreground">
+        No GGUF variants found.
+      </div>
+    );
+  }
+
+  return (
+    <div className="pl-4 border-l-2 border-accent/50 ml-3 my-1">
+      <div className="px-2 py-1 flex items-center gap-1.5">
+        <span className="text-[10px] font-semibold uppercase tracking-wider text-muted-foreground">
+          Quantizations
+        </span>
+        {hasVision && (
+          <span className="text-[9px] font-medium text-blue-400">Vision</span>
+        )}
+      </div>
+      {variants.map((v) => (
+        <button
+          key={v.filename}
+          type="button"
+          onClick={() => handleVariantClick(v.quant)}
+          className={cn(
+            "flex w-full items-center justify-between gap-2 rounded-md px-2.5 py-1 text-left text-sm transition-colors hover:bg-accent",
+          )}
+        >
+          <span className="min-w-0 flex-1 truncate font-mono text-xs">
+            {v.quant}
+            {v.quant === defaultVariant && (
+              <span className="ml-1.5 text-[9px] font-sans font-medium text-primary/70">
+                recommended
+              </span>
+            )}
+          </span>
+          <span className="text-[10px] text-muted-foreground shrink-0">
+            {formatBytes(v.size_bytes)}
+          </span>
+        </button>
+      ))}
+    </div>
+  );
+}
+
+// ── Detect GGUF repos by naming convention ────────────────────
+
+function isGgufRepo(id: string): boolean {
+  return id.toUpperCase().includes("-GGUF");
+}
+
+// ── Hub Model Picker ──────────────────────────────────────────
+
 export function HubModelPicker({
   models,
   value,
@@ -130,6 +259,9 @@ export function HubModelPicker({
     debouncedQuery,
   );
 
+  // Track which GGUF repo is expanded for variant selection
+  const [expandedGguf, setExpandedGguf] = useState<string | null>(null);
+
   const recommendedIds = useMemo(
     () => dedupe([...models.map((model) => model.id), value ?? ""]),
     [models, value],
@@ -199,6 +331,19 @@ export function HubModelPicker({
 
   const { scrollRef, sentinelRef } = useInfiniteScroll(fetchMore, results.length);
 
+  /** Handle clicking a model row — GGUF repos expand, others load directly. */
+  const handleModelClick = useCallback(
+    (id: string) => {
+      if (isGgufRepo(id)) {
+        // Toggle GGUF variant expander
+        setExpandedGguf((prev) => (prev === id ? null : id));
+      } else {
+        onSelect(id, { source: "hub", isLora: false });
+      }
+    },
+    [onSelect],
+  );
+
   return (
     <div className="space-y-2">
       <div className="relative">
@@ -230,18 +375,24 @@ export function HubModelPicker({
                 recommendedIds.map((id) => {
                   const vram = recommendedVramMap.get(id);
                   return (
-                    <ModelRow
-                      key={id}
-                      label={id}
-                      meta={vram?.detail ?? undefined}
-                      selected={value === id}
-                      onClick={() =>
-                        onSelect(id, { source: "hub", isLora: false })
-                      }
-                      vramStatus={vram?.status ?? null}
-                      vramEst={vram?.est}
-                      gpuGb={gpu.available ? gpu.memoryTotalGb : undefined}
-                    />
+                    <div key={id}>
+                      <ModelRow
+                        label={id}
+                        meta={
+                          isGgufRepo(id)
+                            ? "GGUF"
+                            : vram?.detail ?? undefined
+                        }
+                        selected={value === id}
+                        onClick={() => handleModelClick(id)}
+                        vramStatus={isGgufRepo(id) ? null : vram?.status ?? null}
+                        vramEst={isGgufRepo(id) ? undefined : vram?.est}
+                        gpuGb={gpu.available ? gpu.memoryTotalGb : undefined}
+                      />
+                      {expandedGguf === id && (
+                        <GgufVariantExpander repoId={id} onSelect={onSelect} />
+                      )}
+                    </div>
                   );
                 })
               )}
@@ -259,18 +410,24 @@ export function HubModelPicker({
                 hfIds.map((id) => {
                   const vram = vramMap.get(id);
                   return (
-                    <ModelRow
-                      key={id}
-                      label={id}
-                      meta={metricsById.get(id)}
-                      selected={value === id}
-                      onClick={() =>
-                        onSelect(id, { source: "hub", isLora: false })
-                      }
-                      vramStatus={vram?.status ?? null}
-                      vramEst={vram?.est}
-                      gpuGb={gpu.available ? gpu.memoryTotalGb : undefined}
-                    />
+                    <div key={id}>
+                      <ModelRow
+                        label={id}
+                        meta={
+                          isGgufRepo(id)
+                            ? "GGUF"
+                            : metricsById.get(id)
+                        }
+                        selected={value === id}
+                        onClick={() => handleModelClick(id)}
+                        vramStatus={isGgufRepo(id) ? null : vram?.status ?? null}
+                        vramEst={isGgufRepo(id) ? undefined : vram?.est}
+                        gpuGb={gpu.available ? gpu.memoryTotalGb : undefined}
+                      />
+                      {expandedGguf === id && (
+                        <GgufVariantExpander repoId={id} onSelect={onSelect} />
+                      )}
+                    </div>
                   );
                 })
               )}
@@ -382,4 +539,3 @@ export function LoraModelPicker({
     </div>
   );
 }
-
diff --git a/studio/frontend/src/components/assistant-ui/model-selector/types.ts b/studio/frontend/src/components/assistant-ui/model-selector/types.ts
index dcf110bfb7..a94d3dd931 100644
--- a/studio/frontend/src/components/assistant-ui/model-selector/types.ts
+++ b/studio/frontend/src/components/assistant-ui/model-selector/types.ts
@@ -15,5 +15,6 @@ export interface LoraModelOption extends ModelOption {
 export interface ModelSelectorChangeMeta {
   source: "hub" | "lora";
   isLora: boolean;
+  ggufVariant?: string;
 }
 
diff --git a/studio/frontend/src/features/chat/api/chat-api.ts b/studio/frontend/src/features/chat/api/chat-api.ts
index 72baf9a6f6..5d5a9551ef 100644
--- a/studio/frontend/src/features/chat/api/chat-api.ts
+++ b/studio/frontend/src/features/chat/api/chat-api.ts
@@ -1,5 +1,6 @@
 import { authFetch } from "@/features/auth";
 import type {
+  GgufVariantsResponse,
   InferenceStatusResponse,
   ListLorasResponse,
   ListModelsResponse,
@@ -74,6 +75,16 @@ export async function unloadModel(payload: UnloadModelRequest): Promise<void> {
   await parseJsonOrThrow<unknown>(response);
 }
 
+export async function listGgufVariants(
+  repoId: string,
+  hfToken?: string,
+): Promise<GgufVariantsResponse> {
+  const params = new URLSearchParams({ repo_id: repoId });
+  if (hfToken) params.set("hf_token", hfToken);
+  const response = await authFetch(`/api/models/gguf-variants?${params}`);
+  return parseJsonOrThrow<GgufVariantsResponse>(response);
+}
+
 function parseSseEvent(rawEvent: string): string[] {
   const dataLines: string[] = [];
   for (const line of rawEvent.split(/\r?\n/)) {
diff --git a/studio/frontend/src/features/chat/chat-page.tsx b/studio/frontend/src/features/chat/chat-page.tsx
index c363704d61..7c57bdf0a4 100644
--- a/studio/frontend/src/features/chat/chat-page.tsx
+++ b/studio/frontend/src/features/chat/chat-page.tsx
@@ -300,7 +300,7 @@ export function ChatPage(): ReactElement {
   }, [inferenceParams.checkpoint, lorasFromStore]);
 
   const handleCheckpointChange = useCallback(
-    (value: string, meta?: { isLora: boolean }) => {
+    (value: string, meta?: { isLora: boolean; ggufVariant?: string }) => {
       const currentCheckpoint =
         useChatRuntimeStore.getState().params.checkpoint;
       if (!value || value === currentCheckpoint) return;
@@ -309,7 +309,11 @@ export function ChatPage(): ReactElement {
         if (currentCheckpoint) {
           await ejectModel();
         }
-        await selectModel({ id: value, isLora: meta?.isLora });
+        await selectModel({
+          id: value,
+          isLora: meta?.isLora,
+          ggufVariant: meta?.ggufVariant,
+        });
       })();
     },
     [selectModel, ejectModel],
diff --git a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
index 030fe55bff..24a6b3c01f 100644
--- a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
+++ b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
@@ -20,6 +20,7 @@ const DEFAULT_MODEL_MAX_SEQ_LENGTH = 2048;
 type SelectedModelInput = {
   id: string;
   isLora?: boolean;
+  ggufVariant?: string;
 };
 
 const LORA_SUFFIX_RE = /_(\d{9,})$/;
@@ -159,6 +160,8 @@ export function useChatModelRuntime() {
 
       const explicitIsLora =
         typeof selection === "string" ? undefined : selection.isLora;
+      const ggufVariant =
+        typeof selection === "string" ? undefined : selection.ggufVariant;
       const model = models.find((entry) => entry.id === modelId);
       const lora = loras.find((entry) => entry.id === modelId);
       const isLora =
@@ -181,6 +184,7 @@ export function useChatModelRuntime() {
             max_seq_length: DEFAULT_MODEL_MAX_SEQ_LENGTH,
             load_in_4bit: true,
             is_lora: isLora,
+            gguf_variant: ggufVariant ?? null,
           });
 
           const currentParams = useChatRuntimeStore.getState().params;
diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.ts
index bc9268e2c2..f0a10a6cae 100644
--- a/studio/frontend/src/features/chat/types/api.ts
+++ b/studio/frontend/src/features/chat/types/api.ts
@@ -28,6 +28,20 @@ export interface LoadModelRequest {
   max_seq_length: number;
   load_in_4bit: boolean;
   is_lora: boolean;
+  gguf_variant?: string | null;
+}
+
+export interface GgufVariantDetail {
+  filename: string;
+  quant: string;
+  size_bytes: number;
+}
+
+export interface GgufVariantsResponse {
+  repo_id: string;
+  variants: GgufVariantDetail[];
+  has_vision: boolean;
+  default_variant: string | null;
 }
 
 export interface LoadModelResponse {

From 2ebeba8588f376529a221a390d19fef4a6139a79 Mon Sep 17 00:00:00 2001
From: Roland Tannous <115670425+rolandtannous@users.noreply.github.com>
Date: Tue, 24 Feb 2026 19:21:01 +0400
Subject: [PATCH 07/12] Switch GGUF backend from /v1/completions to
 /v1/chat/completions

Fixes two bugs:
1. Chat template tags (<|im_start|>, <|im_end|>) leaking into output
   because /v1/completions treated them as literal text
2. Image hallucination because image_b64 was never passed to llama-server

Now llama-server handles chat templates natively and receives images
as OpenAI-format multimodal content parts for vision models.
---
 studio/backend/core/inference/llama_cpp.py | 133 +++++++--------------
 studio/backend/routes/inference.py         |  12 +-
 2 files changed, 51 insertions(+), 94 deletions(-)

diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index b5c7d3c50d..30fe660404 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -2,7 +2,7 @@
 llama-server inference backend for GGUF models.
 
 Manages a llama-server subprocess and proxies chat completions
-through its /v1/completions endpoint.
+through its OpenAI-compatible /v1/chat/completions endpoint.
 """
 import atexit
 import json
@@ -27,7 +27,7 @@ class LlamaCppBackend:
 
     Lifecycle:
         1. load_model()  — starts llama-server with the GGUF file
-        2. generate_chat_completion() — formats prompt, proxies to /v1/completions, streams back
+        2. generate_chat_completion() — proxies to /v1/chat/completions, streams back
         3. unload_model() — terminates llama-server subprocess
     """
 
@@ -41,7 +41,6 @@ class LlamaCppBackend:
         self._is_vision: bool = False
         self._healthy = False
         self._lock = threading.Lock()
-        self._chat_template: Optional[str] = None
 
         atexit.register(self._cleanup)
 
@@ -218,13 +217,6 @@ class LlamaCppBackend:
 
             self._healthy = True
 
-            # Read chat template from local GGUF metadata (skip in HF mode —
-            # llama-server handles template application internally)
-            if gguf_path:
-                self._chat_template = self._read_gguf_chat_template(gguf_path)
-            else:
-                self._chat_template = None
-
             logger.info(
                 f"llama-server ready on port {self._port} "
                 f"for model '{model_identifier}'"
@@ -243,7 +235,6 @@ class LlamaCppBackend:
             self._is_vision = False
             self._port = None
             self._healthy = False
-            self._chat_template = None
             return True
 
     def _kill_process(self):
@@ -298,92 +289,49 @@ class LlamaCppBackend:
         logger.error(f"llama-server health check timed out after {timeout}s")
         return False
 
-    # ── Chat template ─────────────────────────────────────────────
+    # ── Message building (OpenAI format) ──────────────────────────
 
     @staticmethod
-    def _read_gguf_chat_template(gguf_path: str) -> Optional[str]:
+    def _build_openai_messages(
+        messages: list[dict],
+        image_b64: Optional[str] = None,
+    ) -> list[dict]:
         """
-        Try to read the chat_template from GGUF file metadata.
+        Build OpenAI-format messages, optionally injecting an image_url
+        content part into the last user message for vision models.
 
-        Uses the gguf Python library if available.
-        Returns the Jinja2 template string, or None.
+        If no image is provided, returns messages as-is.
         """
-        try:
-            from gguf import GGUFReader
+        if not image_b64:
+            return messages
 
-            reader = GGUFReader(gguf_path)
-            for field_name in reader.fields:
-                if field_name == "tokenizer.chat_template":
-                    field = reader.fields[field_name]
-                    # Field data is an array of bytes
-                    template_bytes = bytes(field.parts[field.data[0]])
-                    template = template_bytes.decode("utf-8")
-                    logger.info(f"Read chat template from GGUF metadata ({len(template)} chars)")
-                    return template
-        except ImportError:
-            logger.debug("gguf library not available, cannot read chat template from GGUF metadata")
-        except Exception as e:
-            logger.warning(f"Could not read chat template from GGUF: {e}")
+        # Find the last user message and convert to multimodal content parts
+        result = [msg.copy() for msg in messages]
+        last_user_idx = None
+        for i, msg in enumerate(result):
+            if msg["role"] == "user":
+                last_user_idx = i
 
-        return None
+        if last_user_idx is not None:
+            text_content = result[last_user_idx].get("content", "")
+            result[last_user_idx]["content"] = [
+                {"type": "text", "text": text_content},
+                {
+                    "type": "image_url",
+                    "image_url": {
+                        "url": f"data:image/png;base64,{image_b64}",
+                    },
+                },
+            ]
 
-    def format_prompt(self, messages: list[dict], system_prompt: str = "") -> str:
-        """
-        Format chat messages into a raw prompt string for /v1/completions.
-
-        Attempts to:
-        1. Render the GGUF's embedded chat_template with Jinja2
-        2. Fallback to ChatML format
-        """
-        # Build full message list with system prompt
-        full_messages = []
-        if system_prompt:
-            full_messages.append({"role": "system", "content": system_prompt})
-        full_messages.extend(messages)
-
-        # Try Jinja2 rendering if we have a template
-        if self._chat_template:
-            try:
-                return self._render_jinja_template(full_messages)
-            except Exception as e:
-                logger.warning(f"Jinja2 template rendering failed, falling back to ChatML: {e}")
-
-        # Fallback: ChatML format
-        return self._format_chatml(full_messages)
-
-    def _render_jinja_template(self, messages: list[dict]) -> str:
-        """Render messages using the GGUF's Jinja2 chat template."""
-        from jinja2 import BaseLoader, Environment
-
-        env = Environment(loader=BaseLoader(), keep_trailing_newline=True)
-        # Add common template globals
-        env.globals["raise_exception"] = lambda msg: (_ for _ in ()).throw(ValueError(msg))
-
-        template = env.from_string(self._chat_template)
-        rendered = template.render(
-            messages=messages,
-            add_generation_prompt=True,
-            bos_token="<s>",
-            eos_token="</s>",
-        )
-        return rendered
-
-    @staticmethod
-    def _format_chatml(messages: list[dict]) -> str:
-        """Format messages using ChatML template (universal fallback)."""
-        parts = []
-        for msg in messages:
-            role = msg.get("role", "user")
-            content = msg.get("content", "")
-            parts.append(f"<|im_start|>{role}\n{content}<|im_end|>")
-        parts.append("<|im_start|>assistant")
-        return "\n".join(parts) + "\n"
+        return result
 
     # ── Generation (proxy to llama-server) ────────────────────────
 
     def generate_chat_completion(
         self,
-        prompt: str,
+        messages: list[dict],
+        image_b64: Optional[str] = None,
         temperature: float = 0.7,
         top_p: float = 0.9,
         top_k: int = 40,
@@ -394,30 +342,32 @@ class LlamaCppBackend:
         cancel_event: Optional[threading.Event] = None,
     ) -> Generator[str, None, None]:
         """
-        Send a completion request to llama-server and stream tokens back.
+        Send a chat completion request to llama-server and stream tokens back.
 
-        Uses /v1/completions (NOT /v1/chat/completions) so we control
-        the prompt format entirely.
+        Uses /v1/chat/completions — llama-server handles chat template
+        application and vision (multimodal image_url parts) natively.
 
         Yields cumulative text (matching InferenceBackend's convention).
         """
         if not self.is_loaded:
             raise RuntimeError("llama-server is not loaded")
 
+        openai_messages = self._build_openai_messages(messages, image_b64)
+
         payload = {
-            "prompt": prompt,
+            "messages": openai_messages,
             "stream": True,
             "temperature": temperature,
             "top_p": top_p,
             "top_k": top_k if top_k >= 0 else 0,
             "min_p": min_p,
-            "n_predict": max_tokens,
+            "max_tokens": max_tokens,
             "repeat_penalty": repetition_penalty,
         }
         if stop:
             payload["stop"] = stop
 
-        url = f"{self.base_url}/v1/completions"
+        url = f"{self.base_url}/v1/chat/completions"
         cumulative = ""
 
         try:
@@ -450,7 +400,8 @@ class LlamaCppBackend:
                                 data = json.loads(line[6:])
                                 choices = data.get("choices", [])
                                 if choices:
-                                    token = choices[0].get("text", "")
+                                    delta = choices[0].get("delta", {})
+                                    token = delta.get("content", "")
                                     if token:
                                         cumulative += token
                                         yield cumulative
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index b7a7a33b43..bc450add0d 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -438,7 +438,7 @@ async def openai_chat_completions(payload: ChatCompletionRequest, request: Reque
             detail="At least one non-system message is required.",
         )
 
-    # ── GGUF path: format prompt → proxy to llama-server ──────
+    # ── GGUF path: proxy to llama-server /v1/chat/completions ──
     if using_gguf:
         # Reject images if this GGUF model doesn't support vision
         image_b64 = extracted_image_b64 or payload.image_base64
@@ -448,7 +448,12 @@ async def openai_chat_completions(payload: ChatCompletionRequest, request: Reque
                 detail="Image provided but current GGUF model does not support vision.",
             )
 
-        prompt = llama_backend.format_prompt(chat_messages, system_prompt)
+        # Build message list with system prompt prepended
+        gguf_messages = []
+        if system_prompt:
+            gguf_messages.append({"role": "system", "content": system_prompt})
+        gguf_messages.extend(chat_messages)
+
         cancel_event = threading.Event()
 
         completion_id = f"chatcmpl-{uuid.uuid4().hex[:12]}"
@@ -456,7 +461,8 @@ async def openai_chat_completions(payload: ChatCompletionRequest, request: Reque
 
         def gguf_generate():
             return llama_backend.generate_chat_completion(
-                prompt=prompt,
+                messages=gguf_messages,
+                image_b64=image_b64,
                 temperature=payload.temperature,
                 top_p=payload.top_p,
                 top_k=payload.top_k,

From 9e280eb105139ed79b017a1499987de1c97b3d8d Mon Sep 17 00:00:00 2001
From: Roland Tannous <115670425+rolandtannous@users.noreply.github.com>
Date: Wed, 25 Feb 2026 03:30:54 +0400
Subject: [PATCH 08/12] Fix GGUF export cwd confusion: remove os.chdir, use
 absolute paths

Remove os.chdir(save_directory) from export.py which was causing all of
unsloth-zoo's relative-path internals (check_llama_cpp, use_local_gguf,
_download_convert_hf_to_gguf) to resolve against the export directory
instead of the repo root. This caused llama.cpp to be cloned inside each
export dir and destroyed the repo root's llama-server build on cleanup.

Now passes absolute paths to save_pretrained_gguf so unsloth resolves
llama.cpp from the repo root where setup.sh already built it.

Also builds llama-quantize in setup.sh (needed by unsloth-zoo's export
pipeline) and symlinks it to llama.cpp root for check_llama_cpp().
---
 setup.sh                             | 20 +++++++--
 studio/backend/core/export/export.py | 62 ++++++++++------------------
 2 files changed, 38 insertions(+), 44 deletions(-)

diff --git a/setup.sh b/setup.sh
index 36165dac65..0d8821d50a 100755
--- a/setup.sh
+++ b/setup.sh
@@ -206,10 +206,11 @@ else
     fi
 fi
 
-# ── 8. Build llama-server for GGUF inference ──
+# ── 8. Build llama.cpp binaries for GGUF inference + export ──
 # Builds in-tree at $REPO/llama.cpp/. This directory is shared with
-# unsloth-zoo's GGUF export pipeline — if converter/quantize are missing,
-# unsloth-zoo will rebuild them on first export. We only build llama-server here.
+# unsloth-zoo's GGUF export pipeline. We build:
+#   - llama-server: for GGUF model inference
+#   - llama-quantize: for GGUF export quantization (symlinked to root for check_llama_cpp())
 LLAMA_SERVER_BIN="$SCRIPT_DIR/llama.cpp/build/bin/llama-server"
 if [ -f "$LLAMA_SERVER_BIN" ]; then
     echo ""
@@ -272,12 +273,25 @@ else
             run_quiet "build llama-server" cmake --build "$LLAMA_CPP_DIR/build" --config Release --target llama-server -j"$NCPU" || BUILD_OK=false
         fi
 
+        # Also build llama-quantize (needed by unsloth-zoo's GGUF export pipeline)
+        if [ "$BUILD_OK" = true ]; then
+            run_quiet "build llama-quantize" cmake --build "$LLAMA_CPP_DIR/build" --config Release --target llama-quantize -j"$NCPU" || true
+            # Symlink to llama.cpp root — check_llama_cpp() looks for the binary there
+            QUANTIZE_BIN="$LLAMA_CPP_DIR/build/bin/llama-quantize"
+            if [ -f "$QUANTIZE_BIN" ]; then
+                ln -sf build/bin/llama-quantize "$LLAMA_CPP_DIR/llama-quantize"
+            fi
+        fi
+
         if [ "$BUILD_OK" = true ]; then
             if [ -f "$LLAMA_SERVER_BIN" ]; then
                 echo "✅ llama-server built at $LLAMA_SERVER_BIN"
             else
                 echo "⚠️  llama-server binary not found after build — GGUF inference won't be available"
             fi
+            if [ -f "$LLAMA_CPP_DIR/llama-quantize" ]; then
+                echo "✅ llama-quantize available for GGUF export"
+            fi
         else
             echo "⚠️  llama-server build failed — GGUF inference won't be available, but everything else works"
         fi
diff --git a/studio/backend/core/export/export.py b/studio/backend/core/export/export.py
index da5b11c60d..865900cb65 100644
--- a/studio/backend/core/export/export.py
+++ b/studio/backend/core/export/export.py
@@ -378,53 +378,33 @@ class ExportBackend:
 
             # Save locally if requested
             if save_directory:
-                logger.info(f"Saving GGUF model locally to: {save_directory}")
+                # Resolve to absolute path so unsloth's relative-path internals
+                # (check_llama_cpp, use_local_gguf, _download_convert_hf_to_gguf)
+                # all resolve against the repo root cwd, NOT the export directory.
+                abs_save_dir = os.path.abspath(save_directory)
+                logger.info(f"Saving GGUF model locally to: {abs_save_dir}")
 
                 # Create the directory if it doesn't exist
-                os.makedirs(save_directory, exist_ok=True)
+                os.makedirs(abs_save_dir, exist_ok=True)
 
-                # Get the base filename for the GGUF file
-                import shutil
-                original_dir = os.getcwd()
+                # On WSL, patch out sudo check before llama.cpp build
+                _apply_wsl_sudo_patch()
 
-                try:
-                    # Change to target directory
-                    os.chdir(save_directory)
-                    logger.info(f"Changed directory to: {save_directory}")
+                # Enable verbose logging so subprocess errors are printed
+                os.environ["UNSLOTH_ENABLE_LOGGING"] = "1"
 
-                    # On WSL, patch out sudo check before llama.cpp build
-                    _apply_wsl_sudo_patch()
+                # Pass absolute path — no os.chdir needed.
+                # unsloth saves model files into this directory, while
+                # check_llama_cpp("llama.cpp") resolves against cwd (repo root)
+                # where setup.sh already built llama.cpp with quantizer.
+                model_save_path = os.path.join(abs_save_dir, "model")
+                self.current_model.save_pretrained_gguf(
+                    model_save_path,
+                    self.current_tokenizer,
+                    quantization_method=quant_method
+                )
 
-                    # Now save (will save in current directory)
-                    self.current_model.save_pretrained_gguf(
-                        "model",  # Base filename
-                        self.current_tokenizer,
-                        quantization_method=quant_method
-                    )
-
-                    logger.info(f"GGUF model saved successfully in {save_directory}")
-
-                    # Check if llama.cpp directory was created here
-                    llama_cpp_in_target = os.path.join(save_directory, "llama.cpp")
-                    llama_cpp_in_original = os.path.join(original_dir, "llama.cpp")
-
-                    if os.path.exists(llama_cpp_in_target):
-                        logger.info(f"Found llama.cpp directory in {save_directory}")
-
-                        # Remove llama.cpp from original directory if it exists
-                        if os.path.exists(llama_cpp_in_original):
-                            logger.info(f"Removing existing llama.cpp in {original_dir}")
-                            shutil.rmtree(llama_cpp_in_original)
-
-                        # Move llama.cpp back to original directory
-                        logger.info(f"Moving llama.cpp to {original_dir}")
-                        shutil.move(llama_cpp_in_target, llama_cpp_in_original)
-                        logger.info(f"Successfully moved llama.cpp back to original directory")
-
-                finally:
-                    # Always change back to original directory
-                    os.chdir(original_dir)
-                    logger.info(f"Changed back to original directory: {original_dir}")
+                logger.info(f"GGUF model saved successfully in {abs_save_dir}")
 
             # Push to hub if requested
             if push_to_hub:

From d9434fee4aab46766c8a0a1aa6a3a8dc9f09d8f0 Mon Sep 17 00:00:00 2001
From: Roland Tannous <rolandtannous@gravityq.ai>
Date: Wed, 25 Feb 2026 10:29:05 +0000
Subject: [PATCH 09/12] fix: use raw github URL for vision.py patch + add VLM
 processor diagnostic logging

---
 setup.sh                                | 4 ++++
 studio/backend/core/training/trainer.py | 9 +++++++++
 2 files changed, 13 insertions(+)

diff --git a/setup.sh b/setup.sh
index 0d8821d50a..a911d9ed7a 100755
--- a/setup.sh
+++ b/setup.sh
@@ -169,6 +169,10 @@ if [ "$IS_COLAB" = true ]; then
     LLAMA_CPP_DST="$(pip show unsloth-zoo | grep -i '^Location:' | awk '{print $2}')/unsloth_zoo/llama_cpp.py"
     curl -sSL "https://raw.githubusercontent.com/unslothai/unsloth-zoo/refs/heads/main/unsloth_zoo/llama_cpp.py" \
         -o "$LLAMA_CPP_DST"
+    # Patch: override vision.py with fix from unsloth PR: https://github.com/unslothai/unsloth/pull/4091 until next pypi release
+    VISION_DST="$(pip show unsloth | grep -i '^Location:' | awk '{print $2}')/unsloth/vision.py"
+    curl -sSL "https://raw.githubusercontent.com/unslothai/unsloth/80e0108a684c882965a02a8ed851e3473c1145ab/unsloth/models/vision.py" \
+        -o "$VISION_DST"
     echo "   Installing studio dependencies..."
     run_quiet "pip install studio" pip install -r "$SCRIPT_DIR/studio/backend/requirements/studio.txt"
     echo "✅ Python dependencies installed"
diff --git a/studio/backend/core/training/trainer.py b/studio/backend/core/training/trainer.py
index 804baa33c0..c56b4a8a89 100644
--- a/studio/backend/core/training/trainer.py
+++ b/studio/backend/core/training/trainer.py
@@ -173,6 +173,15 @@ class UnslothTrainer:
                     token=hf_token,
                 )
                 logger.info("Loaded vision model")
+
+                # Diagnostic: check if FastVisionModel returned a real Processor or a raw tokenizer
+                from transformers import ProcessorMixin
+                tok = self.tokenizer
+                has_image_proc = isinstance(tok, ProcessorMixin) or hasattr(tok, "image_processor")
+                print(f"\n[VLM Diagnostic] FastVisionModel returned: {type(tok).__name__}")
+                print(f"[VLM Diagnostic] Is ProcessorMixin: {isinstance(tok, ProcessorMixin)}")
+                print(f"[VLM Diagnostic] Has image_processor: {hasattr(tok, 'image_processor')}")
+                print(f"[VLM Diagnostic] Usable as vision processor: {has_image_proc}\n")
             else:
                 # Load text model - returns (model, tokenizer)
                 self.model, self.tokenizer = FastLanguageModel.from_pretrained(

From b0533503f2962d07bc88560867cf672fadf390a5 Mon Sep 17 00:00:00 2001
From: Roland Tannous <rolandtannous@gravityq.ai>
Date: Wed, 25 Feb 2026 11:39:17 +0000
Subject: [PATCH 10/12] added vision.py patch for vision processor from PR#260

---
 setup.sh | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/setup.sh b/setup.sh
index a911d9ed7a..a6d227de2b 100755
--- a/setup.sh
+++ b/setup.sh
@@ -170,7 +170,7 @@ if [ "$IS_COLAB" = true ]; then
     curl -sSL "https://raw.githubusercontent.com/unslothai/unsloth-zoo/refs/heads/main/unsloth_zoo/llama_cpp.py" \
         -o "$LLAMA_CPP_DST"
     # Patch: override vision.py with fix from unsloth PR: https://github.com/unslothai/unsloth/pull/4091 until next pypi release
-    VISION_DST="$(pip show unsloth | grep -i '^Location:' | awk '{print $2}')/unsloth/vision.py"
+    VISION_DST="$(pip show unsloth | grep -i '^Location:' | awk '{print $2}')/unsloth/models/vision.py"
     curl -sSL "https://raw.githubusercontent.com/unslothai/unsloth/80e0108a684c882965a02a8ed851e3473c1145ab/unsloth/models/vision.py" \
         -o "$VISION_DST"
     echo "   Installing studio dependencies..."
@@ -193,6 +193,10 @@ else
     LLAMA_CPP_DST="$(pip show unsloth-zoo | grep -i '^Location:' | awk '{print $2}')/unsloth_zoo/llama_cpp.py"
     curl -sSL "https://raw.githubusercontent.com/unslothai/unsloth-zoo/refs/heads/main/unsloth_zoo/llama_cpp.py" \
         -o "$LLAMA_CPP_DST"
+    # Patch: override vision.py with fix from unsloth PR: https://github.com/unslothai/unsloth/pull/4091 until next pypi release
+    VISION_DST="$(pip show unsloth | grep -i '^Location:' | awk '{print $2}')/unsloth/models/vision.py"
+    curl -sSL "https://raw.githubusercontent.com/unslothai/unsloth/80e0108a684c882965a02a8ed851e3473c1145ab/unsloth/models/vision.py" \
+        -o "$VISION_DST"
     echo "   Installing studio dependencies..."
     run_quiet "pip install studio" pip install -r "$SCRIPT_DIR/studio/backend/requirements/studio.txt"
     echo "✅ Python dependencies installed"

From 5c3a01899c6b967db05805b0c81917f6be10a9e9 Mon Sep 17 00:00:00 2001
From: Roland Tannous <115670425+rolandtannous@users.noreply.github.com>
Date: Wed, 25 Feb 2026 15:47:45 +0400
Subject: [PATCH 11/12] Filter GGUF models from training page model selectors

GGUF models can't be fine-tuned, so hide them from the training/studio
page while keeping them available for inference on the chat page.

- Add "gguf" to EXCLUDED_TAGS in HF model search hook
- Filter local models with .gguf extension or -GGUF in ID
---
 .../studio/sections/model-section.tsx         | 21 ++++++++++++++-----
 .../frontend/src/hooks/use-hf-model-search.ts |  1 +
 2 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/studio/frontend/src/features/studio/sections/model-section.tsx b/studio/frontend/src/features/studio/sections/model-section.tsx
index 0c4a71a8c3..5040e5bd77 100644
--- a/studio/frontend/src/features/studio/sections/model-section.tsx
+++ b/studio/frontend/src/features/studio/sections/model-section.tsx
@@ -170,14 +170,25 @@ export function ModelSection() {
     return ids;
   }, [hfResults, selectedModel]);
 
+  // Filter out GGUF models — they can't be used for training
+  const trainableLocalModels = useMemo(
+    () =>
+      localModels.filter((m) => {
+        if (m.path.endsWith(".gguf")) return false;
+        if (m.id.toLowerCase().includes("-gguf")) return false;
+        return true;
+      }),
+    [localModels],
+  );
+
   const localMetaById = useMemo(() => {
     const map = new Map<string, LocalModelInfo>();
-    for (const model of localModels) map.set(model.id, model);
+    for (const model of trainableLocalModels) map.set(model.id, model);
     return map;
-  }, [localModels]);
+  }, [trainableLocalModels]);
 
   const localResultIds = useMemo(() => {
-    const ids = localModels.map((model) => model.id);
+    const ids = trainableLocalModels.map((model) => model.id);
     const manual = localModelInput.trim();
     if (manual && !ids.includes(manual)) {
       ids.unshift(manual);
@@ -341,8 +352,8 @@ export function ModelSection() {
             <p className="text-[10px] text-red-500">{localModelsError}</p>
           ) : (
             <p className="text-[10px] text-muted-foreground">
-              {localModels.length > 0
-                ? `${localModels.length} local/cached models found`
+              {trainableLocalModels.length > 0
+                ? `${trainableLocalModels.length} local/cached models found`
                 : "No local models found. Enter path manually."}
             </p>
           )}
diff --git a/studio/frontend/src/hooks/use-hf-model-search.ts b/studio/frontend/src/hooks/use-hf-model-search.ts
index 0029f13317..6ba70a4d5c 100644
--- a/studio/frontend/src/hooks/use-hf-model-search.ts
+++ b/studio/frontend/src/hooks/use-hf-model-search.ts
@@ -11,6 +11,7 @@ export interface HfModelResult {
 }
 
 const EXCLUDED_TAGS = new Set([
+  "gguf",
   "gptq",
   "awq",
   "exl2",

From 52738383f9410edf78fb17858970f35f1ebb37dc Mon Sep 17 00:00:00 2001
From: Roland Tannous <115670425+rolandtannous@users.noreply.github.com>
Date: Wed, 25 Feb 2026 16:00:24 +0400
Subject: [PATCH 12/12] Remove UNSLOTH_ENABLE_LOGGING from export pipeline

---
 studio/backend/core/export/export.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/studio/backend/core/export/export.py b/studio/backend/core/export/export.py
index 865900cb65..dbe11ece52 100644
--- a/studio/backend/core/export/export.py
+++ b/studio/backend/core/export/export.py
@@ -390,9 +390,6 @@ class ExportBackend:
                 # On WSL, patch out sudo check before llama.cpp build
                 _apply_wsl_sudo_patch()
 
-                # Enable verbose logging so subprocess errors are printed
-                os.environ["UNSLOTH_ENABLE_LOGGING"] = "1"
-
                 # Pass absolute path — no os.chdir needed.
                 # unsloth saves model files into this directory, while
                 # check_llama_cpp("llama.cpp") resolves against cwd (repo root)