ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge remote-tracking branch 'origin/main' into woa-nvidia-wsl-fallback

Browse source
This commit is contained in:
Daniel Han 2026-06-11 23:05:11 -07:00
commit 2fa4ec504f
2 changed files with 22 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

9
tests/utils/test_rope_scaling_drift.py
View file

@ -300,7 +300,14 @@ def test_object_style_rope_scaling_on_config_delegates_correctly():
expected = _reference_inv_freq(dict_config, "linear")
object_config = _make_config({"rope_type": "linear", "factor": 4.0})
object_config.rope_scaling = FakeLinearRopeScalingConfig()
try:
object_config.rope_scaling = FakeLinearRopeScalingConfig()
except Exception:
pytest.skip(
"transformers strict-validates rope_scaling to dict/RopeParameters/None, "
"so object-style config.rope_scaling (and the delegation retry it "
"exercises) is unreachable on this version."
)
inv_freq, attention_scaling = _compute_config_rope_inv_freq(
object_config, object_config.rope_scaling
)

14
unsloth/models/llama.py
View file

@ -1673,12 +1673,26 @@ def _llama3_inv_freq_from_config(
return torch.where(is_medium, smoothed, scaled)
def _vanilla_inv_freq_from_config(config, device = "cpu"):
"""Unscaled RoPE inv_freq (rope_type 'default'/None), matching the constructor's fallback."""
base = _get_rope_theta(config, default = 10000.0)
dim = getattr(config, "head_dim", None)
if dim is None:
dim = int(config.hidden_size // config.num_attention_heads)
return 1.0 / (base ** (torch.arange(0, dim, 2, dtype = torch.int64, device = device).float() / dim))
def _compute_config_rope_inv_freq(config, rope_scaling):
"""(inv_freq, attention_scaling) per config.rope_scaling via transformers'
ROPE_INIT_FUNCTIONS, with an inline llama3 fallback; (None, 1.0) on failure."""
original_rope_scaling = rope_scaling
rope_scaling = _rope_scaling_as_dict(rope_scaling)
rope_type = rope_scaling.get("rope_type", None) or rope_scaling.get("type", None)
# "default"/unset means unscaled RoPE. transformers >=5 reports
# rope_type="default" for every plain config and dropped "default" from
# ROPE_INIT_FUNCTIONS, so compute it directly instead of warning per load.
if rope_type in (None, "default"):
return _vanilla_inv_freq_from_config(config).to(dtype = torch.float32, device = "cpu"), 1.0
try:
from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONS