studio/mmproj: skip unwanted GGUF values via seek instead of read (#5431)
The previous _skip_gguf_value walked past discarded values with f.read(n), which allocates and immediately drops a Python bytes object. For weight GGUFs that carry tokenizer.ggml.tokens (~150K unicode strings) this wasted ~10 MB of allocation per cold call. Switch the discard path to f.seek(n, 1). The kernel never has to copy the bytes into userspace and Python never allocates. Truncation is now detected on the next read attempt rather than inline (an out-of-range seek on a regular file is legal and the next read returns short). Measured on real downloaded GGUFs (Qwen3.5-4B IQ2_XXS 1.52 GB, bartowski Qwen3.5-4B IQ2_M 1.70 GB, Qwen3.5-4B-MTP IQ2_M 1.94 GB): before: 142 ms cold per weight, ~11 MB read after: 90 ms cold per weight, ~4 MB read Mmproj reads are unaffected (no tokenizer to skip). Cached re-reads remain ~50 microseconds. All 161 in-tree backend tests + 85 isolated sandbox tests pass.
This commit is contained in:
parent
63c6750532
commit
bbd0ba0c25
1 changed files with 10 additions and 7 deletions
|
|
@ -152,7 +152,9 @@ _FIXED_VTYPE_SIZES: Dict[int, int] = {
|
|||
|
||||
|
||||
def _skip_gguf_value(f, vtype: int) -> bool:
|
||||
"""Advance past one GGUF value. False on truncation or unknown type."""
|
||||
"""Advance past one GGUF value. ``f.seek(.., 1)`` past EOF is legal
|
||||
on a regular file so truncation is detected on the next read; we
|
||||
only return False for unknown types or sanity-bound overflow."""
|
||||
if vtype == 8: # STRING
|
||||
slen_bytes = f.read(8)
|
||||
if len(slen_bytes) < 8:
|
||||
|
|
@ -160,7 +162,8 @@ def _skip_gguf_value(f, vtype: int) -> bool:
|
|||
slen = struct.unpack("<Q", slen_bytes)[0]
|
||||
if slen > 1 << 30: # 1 GB sanity bound
|
||||
return False
|
||||
return len(f.read(slen)) == slen
|
||||
f.seek(slen, 1)
|
||||
return True
|
||||
if vtype == 9: # ARRAY
|
||||
head = f.read(12)
|
||||
if len(head) < 12:
|
||||
|
|
@ -176,18 +179,18 @@ def _skip_gguf_value(f, vtype: int) -> bool:
|
|||
slen = struct.unpack("<Q", slen_bytes)[0]
|
||||
if slen > 1 << 30:
|
||||
return False
|
||||
if len(f.read(slen)) != slen:
|
||||
return False
|
||||
f.seek(slen, 1)
|
||||
return True
|
||||
sz = _FIXED_VTYPE_SIZES.get(atype)
|
||||
if sz is None:
|
||||
return False
|
||||
total = sz * alen
|
||||
return len(f.read(total)) == total
|
||||
f.seek(sz * alen, 1)
|
||||
return True
|
||||
sz = _FIXED_VTYPE_SIZES.get(vtype)
|
||||
if sz is None:
|
||||
return False
|
||||
return len(f.read(sz)) == sz
|
||||
f.seek(sz, 1)
|
||||
return True
|
||||
|
||||
|
||||
def is_mmproj_by_metadata(meta: Optional[Dict[str, str]]) -> Optional[bool]:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue