Check nested precision dicts in the duplicate key guard

The registry nests a per-precision dict ("16" / "8") under 26 entries and
mapper.py reads those keys directly, so a duplicate there overwrites the
earlier mapping exactly like a top level duplicate. The guard only looked at
the top level keys. Walk every dict literal in the registry, count keys per
dict so "16" and "8" repeating across sibling entries stay legal, and report
the offending line numbers so a failure points straight at the entry.
This commit is contained in:
danielhanchen 2026-07-29 09:13:18 +00:00
commit e458c3929e

View file

@ -9,7 +9,6 @@ We inspect the source with ``ast`` to ensure there are no duplicates.
import ast
import os
from collections import Counter
MAPPER_PATH = os.path.join(os.path.dirname(__file__), os.pardir, "unsloth", "models", "mapper.py")
@ -22,18 +21,27 @@ def _duplicate_int_to_float_keys():
if not isinstance(node, ast.Assign):
continue
for target in node.targets:
# The name is mangled at class scope, but at module scope it is the
# plain ``__INT_TO_FLOAT_MAPPER`` identifier.
# ``ast.parse`` stops before code generation, so the identifier is
# never private-name-mangled and reads exactly as written.
if isinstance(target, ast.Name) and target.id == "__INT_TO_FLOAT_MAPPER":
if not isinstance(node.value, ast.Dict):
continue
keys = [
k.value
for k in node.value.keys
if isinstance(k, ast.Constant) and isinstance(k.value, str)
]
counts = Counter(keys)
return {key: n for key, n in counts.items() if n > 1}
# An entry may nest a per-precision dict ("16" / "8") that
# mapper.py reads directly, so every dict literal in the
# registry is checked. Keys are counted per dict, since "16"
# and "8" legitimately repeat across sibling entries.
duplicates = {}
for mapping in ast.walk(node.value):
if not isinstance(mapping, ast.Dict):
continue
seen = set()
for k in mapping.keys:
if not (isinstance(k, ast.Constant) and isinstance(k.value, str)):
continue
if k.value in seen:
duplicates.setdefault(k.value, []).append(k.lineno)
seen.add(k.value)
return duplicates
raise AssertionError("Could not find the __INT_TO_FLOAT_MAPPER dict literal in mapper.py")
@ -42,5 +50,5 @@ def test_int_to_float_mapper_has_no_duplicate_keys():
assert not duplicates, (
"Duplicate keys in __INT_TO_FLOAT_MAPPER silently overwrite earlier "
"entries and corrupt model resolution. Remove the redundant "
f"literal(s): {duplicates}"
f"literal(s), key -> line number(s) in mapper.py: {duplicates}"
)