Stop false RoPE 'default' warning and fix rope drift gate on transformers 5 (#6223)

* Handle rope_type 'default' on transformers 5 to stop false RoPE warning

transformers 5 reports rope_type="default" for every plain (unscaled) config
and dropped "default" from ROPE_INIT_FUNCTIONS. _compute_config_rope_inv_freq
then did ROPE_INIT_FUNCTIONS["default"], hit KeyError, returned None and logged
"Could not apply RoPE scaling 'default'; long-context generation may degrade"
on every model load. The inv_freq was still correct (the constructor recomputes
vanilla on None), but the warning is a false alarm for unscaled models.

Compute the unscaled inv_freq directly for rope_type "default"/None instead of
going through ROPE_INIT_FUNCTIONS, so plain configs return the right value with
no warning. Scaled types (llama3/linear/yarn/...) are unchanged.

Also skip test_object_style_rope_scaling_on_config_delegates_correctly when
transformers strict-validates rope_scaling (5.x): it rejects a non-dict object
on config.rope_scaling, so the object-style delegation path cannot be set up
there. The test still runs and asserts on transformers <5.

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

tests/utils/test_rope_scaling_drift.py

@@ -300,7 +300,14 @@ def test_object_style_rope_scaling_on_config_delegates_correctly():
     expected = _reference_inv_freq(dict_config, "linear")
 
     object_config = _make_config({"rope_type": "linear", "factor": 4.0})
-    object_config.rope_scaling = FakeLinearRopeScalingConfig()
+    try:
+        object_config.rope_scaling = FakeLinearRopeScalingConfig()
+    except Exception:
+        pytest.skip(
+            "transformers strict-validates rope_scaling to dict/RopeParameters/None, "
+            "so object-style config.rope_scaling (and the delegation retry it "
+            "exercises) is unreachable on this version."
+        )
     inv_freq, attention_scaling = _compute_config_rope_inv_freq(
         object_config, object_config.rope_scaling
     )

unsloth/models/llama.py

@@ -1673,12 +1673,26 @@ def _llama3_inv_freq_from_config(
     return torch.where(is_medium, smoothed, scaled)
 
 
+def _vanilla_inv_freq_from_config(config, device = "cpu"):
+    """Unscaled RoPE inv_freq (rope_type 'default'/None), matching the constructor's fallback."""
+    base = _get_rope_theta(config, default = 10000.0)
+    dim = getattr(config, "head_dim", None)
+    if dim is None:
+        dim = int(config.hidden_size // config.num_attention_heads)
+    return 1.0 / (base ** (torch.arange(0, dim, 2, dtype = torch.int64, device = device).float() / dim))
+
+
 def _compute_config_rope_inv_freq(config, rope_scaling):
     """(inv_freq, attention_scaling) per config.rope_scaling via transformers'
     ROPE_INIT_FUNCTIONS, with an inline llama3 fallback; (None, 1.0) on failure."""
     original_rope_scaling = rope_scaling
     rope_scaling = _rope_scaling_as_dict(rope_scaling)
     rope_type = rope_scaling.get("rope_type", None) or rope_scaling.get("type", None)
+    # "default"/unset means unscaled RoPE. transformers >=5 reports
+    # rope_type="default" for every plain config and dropped "default" from
+    # ROPE_INIT_FUNCTIONS, so compute it directly instead of warning per load.
+    if rope_type in (None, "default"):
+        return _vanilla_inv_freq_from_config(config).to(dtype = torch.float32, device = "cpu"), 1.0
     try:
         from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONS