3ce187da02Raise ruff line-length to 100 and extend the local pre-commit format pipeline (def-signature magic-comma normalization, short multi-line assert collapse, kwarg '=' spacing, blank-line-after-short-import removal, adjacent string-literal / f-string+plain merge, redundant-pass pruning). Every transform re-checks the file AST and is dropped if it would differ; the whole-repo reformat is verified AST-identical per file and idempotent.
103 lines
3.6 KiB
Python
103 lines
3.6 KiB
Python
import ast
|
|
import json
|
|
import os
|
|
import types
|
|
from pathlib import Path
|
|
|
|
|
|
class _Logger:
|
|
def warning_once(self, *args, **kwargs):
|
|
pass
|
|
|
|
|
|
def _load_preserve_helper():
|
|
source = Path(__file__).parents[2] / "unsloth" / "save.py"
|
|
tree = ast.parse(source.read_text(encoding = "utf-8"))
|
|
helper = next(
|
|
node
|
|
for node in tree.body
|
|
if isinstance(node, ast.FunctionDef) and node.name == "_preserve_tokenizer_eos_token"
|
|
)
|
|
module = ast.Module(body = [helper], type_ignores = [])
|
|
ast.fix_missing_locations(module)
|
|
namespace = {"json": json, "os": os, "logger": _Logger()}
|
|
exec(compile(module, str(source), "exec"), namespace)
|
|
return namespace["_preserve_tokenizer_eos_token"]
|
|
|
|
|
|
def test_preserve_tokenizer_eos_token_restores_gemma4_turn_token(tmp_path):
|
|
preserve = _load_preserve_helper()
|
|
tokenizer_config = tmp_path / "tokenizer_config.json"
|
|
tokenizer_config.write_text(
|
|
json.dumps({"eos_token": "<eos>", "other": True}),
|
|
encoding = "utf-8",
|
|
)
|
|
tokenizer = types.SimpleNamespace(eos_token = "<turn|>")
|
|
|
|
preserve(tokenizer, tmp_path)
|
|
|
|
saved_config = json.loads(tokenizer_config.read_text(encoding = "utf-8"))
|
|
assert saved_config["eos_token"] == "<turn|>"
|
|
assert saved_config["other"] is True
|
|
|
|
|
|
def test_preserve_tokenizer_eos_token_supports_processor_tokenizer(tmp_path):
|
|
preserve = _load_preserve_helper()
|
|
tokenizer_config = tmp_path / "tokenizer_config.json"
|
|
tokenizer_config.write_text(json.dumps({"eos_token": "<eos>"}), encoding = "utf-8")
|
|
processor = types.SimpleNamespace(tokenizer = types.SimpleNamespace(eos_token = "<turn|>"))
|
|
|
|
preserve(processor, tmp_path)
|
|
|
|
saved_config = json.loads(tokenizer_config.read_text(encoding = "utf-8"))
|
|
assert saved_config["eos_token"] == "<turn|>"
|
|
|
|
|
|
class _StringableToken:
|
|
def __str__(self):
|
|
return "<turn|>"
|
|
|
|
|
|
def test_preserve_tokenizer_eos_token_serializes_stringable_tokens(tmp_path):
|
|
preserve = _load_preserve_helper()
|
|
tokenizer_config = tmp_path / "tokenizer_config.json"
|
|
tokenizer_config.write_text(json.dumps({"eos_token": "<eos>"}), encoding = "utf-8")
|
|
tokenizer = types.SimpleNamespace(eos_token = _StringableToken())
|
|
|
|
preserve(tokenizer, tmp_path)
|
|
|
|
saved_config = json.loads(tokenizer_config.read_text(encoding = "utf-8"))
|
|
assert saved_config["eos_token"] == "<turn|>"
|
|
|
|
|
|
def test_preserve_tokenizer_eos_token_supports_filename_prefix(tmp_path):
|
|
preserve = _load_preserve_helper()
|
|
prefixed_config = tmp_path / "adapter-tokenizer_config.json"
|
|
prefixed_config.write_text(
|
|
json.dumps({"eos_token": "<eos>", "other": True}),
|
|
encoding = "utf-8",
|
|
)
|
|
tokenizer = types.SimpleNamespace(eos_token = "<turn|>")
|
|
|
|
preserve(tokenizer, tmp_path, filename_prefix = "adapter")
|
|
|
|
saved_config = json.loads(prefixed_config.read_text(encoding = "utf-8"))
|
|
assert saved_config["eos_token"] == "<turn|>"
|
|
assert saved_config["other"] is True
|
|
# Unprefixed file must not be created as a side effect.
|
|
assert not (tmp_path / "tokenizer_config.json").exists()
|
|
|
|
|
|
def test_preserve_tokenizer_eos_token_filename_prefix_none_uses_default(tmp_path):
|
|
preserve = _load_preserve_helper()
|
|
default_config = tmp_path / "tokenizer_config.json"
|
|
default_config.write_text(
|
|
json.dumps({"eos_token": "<eos>"}),
|
|
encoding = "utf-8",
|
|
)
|
|
tokenizer = types.SimpleNamespace(eos_token = "<turn|>")
|
|
|
|
preserve(tokenizer, tmp_path, filename_prefix = None)
|
|
|
|
saved_config = json.loads(default_config.read_text(encoding = "utf-8"))
|
|
assert saved_config["eos_token"] == "<turn|>"
|