From 6dae2f525b5218143f85cc8f401313e182bccac3 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 11 Jun 2026 20:37:01 -0700 Subject: [PATCH] Stop false RoPE 'default' warning and fix rope drift gate on transformers 5 (#6223) * Handle rope_type 'default' on transformers 5 to stop false RoPE warning transformers 5 reports rope_type="default" for every plain (unscaled) config and dropped "default" from ROPE_INIT_FUNCTIONS. _compute_config_rope_inv_freq then did ROPE_INIT_FUNCTIONS["default"], hit KeyError, returned None and logged "Could not apply RoPE scaling 'default'; long-context generation may degrade" on every model load. The inv_freq was still correct (the constructor recomputes vanilla on None), but the warning is a false alarm for unscaled models. Compute the unscaled inv_freq directly for rope_type "default"/None instead of going through ROPE_INIT_FUNCTIONS, so plain configs return the right value with no warning. Scaled types (llama3/linear/yarn/...) are unchanged. Also skip test_object_style_rope_scaling_on_config_delegates_correctly when transformers strict-validates rope_scaling (5.x): it rejects a non-dict object on config.rope_scaling, so the object-style delegation path cannot be set up there. The test still runs and asserts on transformers <5. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- tests/utils/test_rope_scaling_drift.py | 9 ++++++++- unsloth/models/llama.py | 14 ++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/tests/utils/test_rope_scaling_drift.py b/tests/utils/test_rope_scaling_drift.py index fae5a7d8a8..193cb830f5 100644 --- a/tests/utils/test_rope_scaling_drift.py +++ b/tests/utils/test_rope_scaling_drift.py @@ -300,7 +300,14 @@ def test_object_style_rope_scaling_on_config_delegates_correctly(): expected = _reference_inv_freq(dict_config, "linear") object_config = _make_config({"rope_type": "linear", "factor": 4.0}) - object_config.rope_scaling = FakeLinearRopeScalingConfig() + try: + object_config.rope_scaling = FakeLinearRopeScalingConfig() + except Exception: + pytest.skip( + "transformers strict-validates rope_scaling to dict/RopeParameters/None, " + "so object-style config.rope_scaling (and the delegation retry it " + "exercises) is unreachable on this version." + ) inv_freq, attention_scaling = _compute_config_rope_inv_freq( object_config, object_config.rope_scaling ) diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index 2d31f71ab1..a60edf3fbe 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -1673,12 +1673,26 @@ def _llama3_inv_freq_from_config( return torch.where(is_medium, smoothed, scaled) +def _vanilla_inv_freq_from_config(config, device = "cpu"): + """Unscaled RoPE inv_freq (rope_type 'default'/None), matching the constructor's fallback.""" + base = _get_rope_theta(config, default = 10000.0) + dim = getattr(config, "head_dim", None) + if dim is None: + dim = int(config.hidden_size // config.num_attention_heads) + return 1.0 / (base ** (torch.arange(0, dim, 2, dtype = torch.int64, device = device).float() / dim)) + + def _compute_config_rope_inv_freq(config, rope_scaling): """(inv_freq, attention_scaling) per config.rope_scaling via transformers' ROPE_INIT_FUNCTIONS, with an inline llama3 fallback; (None, 1.0) on failure.""" original_rope_scaling = rope_scaling rope_scaling = _rope_scaling_as_dict(rope_scaling) rope_type = rope_scaling.get("rope_type", None) or rope_scaling.get("type", None) + # "default"/unset means unscaled RoPE. transformers >=5 reports + # rope_type="default" for every plain config and dropped "default" from + # ROPE_INIT_FUNCTIONS, so compute it directly instead of warning per load. + if rope_type in (None, "default"): + return _vanilla_inv_freq_from_config(config).to(dtype = torch.float32, device = "cpu"), 1.0 try: from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONS