studio/mmproj: skip unwanted GGUF values via seek instead of read (#5431)

The previous _skip_gguf_value walked past discarded values with
f.read(n), which allocates and immediately drops a Python bytes
object. For weight GGUFs that carry tokenizer.ggml.tokens (~150K
unicode strings) this wasted ~10 MB of allocation per cold call.

Switch the discard path to f.seek(n, 1). The kernel never has to
copy the bytes into userspace and Python never allocates. Truncation
is now detected on the next read attempt rather than inline (an
out-of-range seek on a regular file is legal and the next read
returns short).

Measured on real downloaded GGUFs (Qwen3.5-4B IQ2_XXS 1.52 GB,
bartowski Qwen3.5-4B IQ2_M 1.70 GB, Qwen3.5-4B-MTP IQ2_M 1.94 GB):

  before:  142 ms cold per weight, ~11 MB read
  after:    90 ms cold per weight, ~4 MB read

Mmproj reads are unaffected (no tokenizer to skip). Cached re-reads
remain ~50 microseconds. All 161 in-tree backend tests + 85 isolated
sandbox tests pass.
This commit is contained in:
Daniel Han 2026-05-14 21:57:04 -07:00 committed by GitHub
commit bbd0ba0c25
No known key found for this signature in database
GPG key ID: B5690EEEBB952194

View file

@ -152,7 +152,9 @@ _FIXED_VTYPE_SIZES: Dict[int, int] = {
def _skip_gguf_value(f, vtype: int) -> bool:
"""Advance past one GGUF value. False on truncation or unknown type."""
"""Advance past one GGUF value. ``f.seek(.., 1)`` past EOF is legal
on a regular file so truncation is detected on the next read; we
only return False for unknown types or sanity-bound overflow."""
if vtype == 8: # STRING
slen_bytes = f.read(8)
if len(slen_bytes) < 8:
@ -160,7 +162,8 @@ def _skip_gguf_value(f, vtype: int) -> bool:
slen = struct.unpack("<Q", slen_bytes)[0]
if slen > 1 << 30: # 1 GB sanity bound
return False
return len(f.read(slen)) == slen
f.seek(slen, 1)
return True
if vtype == 9: # ARRAY
head = f.read(12)
if len(head) < 12:
@ -176,18 +179,18 @@ def _skip_gguf_value(f, vtype: int) -> bool:
slen = struct.unpack("<Q", slen_bytes)[0]
if slen > 1 << 30:
return False
if len(f.read(slen)) != slen:
return False
f.seek(slen, 1)
return True
sz = _FIXED_VTYPE_SIZES.get(atype)
if sz is None:
return False
total = sz * alen
return len(f.read(total)) == total
f.seek(sz * alen, 1)
return True
sz = _FIXED_VTYPE_SIZES.get(vtype)
if sz is None:
return False
return len(f.read(sz)) == sz
f.seek(sz, 1)
return True
def is_mmproj_by_metadata(meta: Optional[Dict[str, str]]) -> Optional[bool]: