From defa761fb2ec7ebb3626d6befb1abdc6c02779e5 Mon Sep 17 00:00:00 2001 From: Roland Tannous Date: Tue, 10 Mar 2026 19:04:10 +0000 Subject: [PATCH 1/3] fix: download all GGUF shards for split models (e.g. 7B Q8_0) LlamaCppBackend.load_model() only downloaded the first matching GGUF file. For split models (e.g. 7B Q8_0 with 3 shards), llama-server needs all shards present. Now collects and downloads all matching files. --- studio/backend/core/inference/llama_cpp.py | 25 +++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 37a57e4c39..b394284532 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -249,17 +249,22 @@ class LlamaCppBackend: ) # Determine the filename from the variant (e.g., "Q4_K_M" -> find matching file) + # For split GGUFs (e.g., *-00001-of-00003.gguf) we must download ALL shards. gguf_filename = None + gguf_extra_shards: list[str] = [] if hf_variant: # Try common naming patterns try: from huggingface_hub import list_repo_files files = list_repo_files(hf_repo, token=hf_token) variant_lower = hf_variant.lower() - for f in files: - if f.endswith(".gguf") and variant_lower in f.lower(): - gguf_filename = f - break + matching = sorted( + f for f in files + if f.endswith(".gguf") and variant_lower in f.lower() + ) + if matching: + gguf_filename = matching[0] # first shard (or single file) + gguf_extra_shards = matching[1:] # remaining shards if split except Exception as e: logger.warning(f"Could not list repo files: {e}") @@ -269,13 +274,23 @@ class LlamaCppBackend: repo_name = hf_repo.split("/")[-1].replace("-GGUF", "") gguf_filename = f"{repo_name}-{hf_variant}.gguf" - logger.info(f"Downloading GGUF: {hf_repo}/{gguf_filename}") + logger.info(f"Downloading GGUF: {hf_repo}/{gguf_filename}" + + (f" (+{len(gguf_extra_shards)} shards)" if gguf_extra_shards else "")) try: local_path = hf_hub_download( repo_id=hf_repo, filename=gguf_filename, token=hf_token, ) + # Download remaining shards for split GGUFs — llama-server + # auto-discovers them when they are in the same directory. + for shard in gguf_extra_shards: + logger.info(f"Downloading GGUF shard: {shard}") + hf_hub_download( + repo_id=hf_repo, + filename=shard, + token=hf_token, + ) except Exception as e: raise RuntimeError( f"Failed to download GGUF file '{gguf_filename}' from {hf_repo}: {e}" From d635846b8da4809159b1f5f2a591b97bc162df6d Mon Sep 17 00:00:00 2001 From: Roland Tannous Date: Tue, 10 Mar 2026 19:13:03 +0000 Subject: [PATCH 2/3] fix: use exact variant matching and shard-prefix discovery for split GGUFs Substring matching (e.g. "Q8_0" in filename) could match superset variants like "IQ8_0", causing wrong quantizations to be downloaded. Now uses word-boundary regex for variant matching and discovers split shards by shared filename prefix rather than treating all variant matches as shards. --- studio/backend/core/inference/llama_cpp.py | 28 +++++++++++++++++----- 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index b394284532..934062f56b 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -255,16 +255,32 @@ class LlamaCppBackend: if hf_variant: # Try common naming patterns try: + import re from huggingface_hub import list_repo_files files = list_repo_files(hf_repo, token=hf_token) variant_lower = hf_variant.lower() - matching = sorted( - f for f in files - if f.endswith(".gguf") and variant_lower in f.lower() + # Use word-boundary matching so "Q8_0" doesn't also + # match "IQ8_0" or other superset variant names. + boundary = re.compile( + r'(? Date: Tue, 10 Mar 2026 19:28:26 +0000 Subject: [PATCH 3/3] fix: restrict shard siblings to exact basename and total count startswith(prefix) could match unrelated split variants whose names extend the selected file's prefix (e.g. model-Q8_0-v2-00001-of-...). Now builds an exact regex from the chosen file's base prefix and shard total so only true siblings are downloaded. --- studio/backend/core/inference/llama_cpp.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 934062f56b..dac8d380c3 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -271,15 +271,19 @@ class LlamaCppBackend: if gguf_files: gguf_filename = gguf_files[0] # For split GGUFs (e.g. model-Q8_0-00001-of-00003.gguf) - # discover siblings by shared prefix instead of - # trusting all variant matches to be shards. - shard_pat = re.compile(r'^(.*)-\d{5}-of-\d{5}\.gguf$') + # discover siblings by exact basename + total match + # so "model-Q8_0-v2-*" isn't pulled in as a sibling. + shard_pat = re.compile(r'^(.*)-\d{5}-of-(\d{5})\.gguf$') m = shard_pat.match(gguf_filename) if m: prefix = m.group(1) + total = m.group(2) + sibling_pat = re.compile( + r'^' + re.escape(prefix) + r'-\d{5}-of-' + re.escape(total) + r'\.gguf$' + ) gguf_extra_shards = [ f for f in gguf_files[1:] - if f.startswith(prefix + "-") and shard_pat.match(f) + if sibling_pat.match(f) ] except Exception as e: logger.warning(f"Could not list repo files: {e}")