Check nested precision dicts in the duplicate key guard
The registry nests a per-precision dict ("16" / "8") under 26 entries and
mapper.py reads those keys directly, so a duplicate there overwrites the
earlier mapping exactly like a top level duplicate. The guard only looked at
the top level keys. Walk every dict literal in the registry, count keys per
dict so "16" and "8" repeating across sibling entries stay legal, and report
the offending line numbers so a failure points straight at the entry.
This commit is contained in:
parent
b25f1e561b
commit
e458c3929e
1 changed files with 19 additions and 11 deletions
|
|
@ -9,7 +9,6 @@ We inspect the source with ``ast`` to ensure there are no duplicates.
|
|||
|
||||
import ast
|
||||
import os
|
||||
from collections import Counter
|
||||
|
||||
MAPPER_PATH = os.path.join(os.path.dirname(__file__), os.pardir, "unsloth", "models", "mapper.py")
|
||||
|
||||
|
|
@ -22,18 +21,27 @@ def _duplicate_int_to_float_keys():
|
|||
if not isinstance(node, ast.Assign):
|
||||
continue
|
||||
for target in node.targets:
|
||||
# The name is mangled at class scope, but at module scope it is the
|
||||
# plain ``__INT_TO_FLOAT_MAPPER`` identifier.
|
||||
# ``ast.parse`` stops before code generation, so the identifier is
|
||||
# never private-name-mangled and reads exactly as written.
|
||||
if isinstance(target, ast.Name) and target.id == "__INT_TO_FLOAT_MAPPER":
|
||||
if not isinstance(node.value, ast.Dict):
|
||||
continue
|
||||
keys = [
|
||||
k.value
|
||||
for k in node.value.keys
|
||||
if isinstance(k, ast.Constant) and isinstance(k.value, str)
|
||||
]
|
||||
counts = Counter(keys)
|
||||
return {key: n for key, n in counts.items() if n > 1}
|
||||
# An entry may nest a per-precision dict ("16" / "8") that
|
||||
# mapper.py reads directly, so every dict literal in the
|
||||
# registry is checked. Keys are counted per dict, since "16"
|
||||
# and "8" legitimately repeat across sibling entries.
|
||||
duplicates = {}
|
||||
for mapping in ast.walk(node.value):
|
||||
if not isinstance(mapping, ast.Dict):
|
||||
continue
|
||||
seen = set()
|
||||
for k in mapping.keys:
|
||||
if not (isinstance(k, ast.Constant) and isinstance(k.value, str)):
|
||||
continue
|
||||
if k.value in seen:
|
||||
duplicates.setdefault(k.value, []).append(k.lineno)
|
||||
seen.add(k.value)
|
||||
return duplicates
|
||||
raise AssertionError("Could not find the __INT_TO_FLOAT_MAPPER dict literal in mapper.py")
|
||||
|
||||
|
||||
|
|
@ -42,5 +50,5 @@ def test_int_to_float_mapper_has_no_duplicate_keys():
|
|||
assert not duplicates, (
|
||||
"Duplicate keys in __INT_TO_FLOAT_MAPPER silently overwrite earlier "
|
||||
"entries and corrupt model resolution. Remove the redundant "
|
||||
f"literal(s): {duplicates}"
|
||||
f"literal(s), key -> line number(s) in mapper.py: {duplicates}"
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue