Add test to guard against duplicate keys in __INT_TO_FLOAT_MAPPER (#7419)
* Add test to guard against duplicate keys in __INT_TO_FLOAT_MAPPER * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Refactor docstring in __INT_TO_FLOAT_MAPPER * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Name the encoding when reading mapper.py tests/test_source_read_encoding.py requires every test that reads a checked-in file to pass an explicit encoding, because open() with no encoding uses the locale encoding, which is cp1252 on a stock Windows install. Without this the new test fails that guard in the auto discovered "Repo tests (CPU)" job. Matches tests/test_gemma_2b_mapper_key.py, which reads the same file, and adds the SPDX header new files in tests/ carry. * Check nested precision dicts in the duplicate key guard The registry nests a per-precision dict ("16" / "8") under 26 entries and mapper.py reads those keys directly, so a duplicate there overwrites the earlier mapping exactly like a top level duplicate. The guard only looked at the top level keys. Walk every dict literal in the registry, count keys per dict so "16" and "8" repeating across sibling entries stay legal, and report the offending line numbers so a failure points straight at the entry. * Tighten comments in the mapper duplicate key guard --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Lee Jackson <130007945+Imagineer99@users.noreply.github.com> Co-authored-by: Daniel Han <danielhanchen@gmail.com>
This commit is contained in:
parent
3212710a4a
commit
7b211c30fe
2 changed files with 53 additions and 4 deletions
53
tests/test_mapper_no_duplicate_keys.py
Normal file
53
tests/test_mapper_no_duplicate_keys.py
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
|
||||
|
||||
"""Guard against duplicate keys in the ``__INT_TO_FLOAT_MAPPER`` registry.
|
||||
|
||||
Duplicate keys in the dict literal silently overwrite earlier entries.
|
||||
We inspect the source with ``ast`` to ensure there are no duplicates.
|
||||
"""
|
||||
|
||||
import ast
|
||||
import os
|
||||
|
||||
MAPPER_PATH = os.path.join(os.path.dirname(__file__), os.pardir, "unsloth", "models", "mapper.py")
|
||||
|
||||
|
||||
def _duplicate_int_to_float_keys():
|
||||
with open(MAPPER_PATH, encoding = "utf-8") as f:
|
||||
tree = ast.parse(f.read(), MAPPER_PATH)
|
||||
|
||||
for node in ast.walk(tree):
|
||||
if not isinstance(node, ast.Assign):
|
||||
continue
|
||||
for target in node.targets:
|
||||
# Private names are mangled at code generation, which ``ast.parse``
|
||||
# never reaches, so the identifier reads exactly as written.
|
||||
if isinstance(target, ast.Name) and target.id == "__INT_TO_FLOAT_MAPPER":
|
||||
if not isinstance(node.value, ast.Dict):
|
||||
continue
|
||||
# mapper.py reads the nested per-precision dicts directly, so
|
||||
# check every dict. Count per dict: "16" and "8" legitimately
|
||||
# repeat across sibling entries.
|
||||
duplicates = {}
|
||||
for mapping in ast.walk(node.value):
|
||||
if not isinstance(mapping, ast.Dict):
|
||||
continue
|
||||
seen = set()
|
||||
for k in mapping.keys:
|
||||
if not (isinstance(k, ast.Constant) and isinstance(k.value, str)):
|
||||
continue
|
||||
if k.value in seen:
|
||||
duplicates.setdefault(k.value, []).append(k.lineno)
|
||||
seen.add(k.value)
|
||||
return duplicates
|
||||
raise AssertionError("Could not find the __INT_TO_FLOAT_MAPPER dict literal in mapper.py")
|
||||
|
||||
|
||||
def test_int_to_float_mapper_has_no_duplicate_keys():
|
||||
duplicates = _duplicate_int_to_float_keys()
|
||||
assert not duplicates, (
|
||||
"Duplicate keys in __INT_TO_FLOAT_MAPPER silently overwrite earlier "
|
||||
"entries and corrupt model resolution. Remove the redundant "
|
||||
f"literal(s), key -> line number(s) in mapper.py: {duplicates}"
|
||||
)
|
||||
Loading…
Add table
Add a link
Reference in a new issue