ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[pre-commit.ci] auto fixes from pre-commit.com hooks

Browse source 
for more information, see https://pre-commit.ci
This commit is contained in:
pre-commit-ci[bot] 2026-05-31 03:43:23 +00:00
commit cca441f125
1 changed files with 77 additions and 24 deletions
Download patch file Download diff file Expand all files Collapse all files

101
tests/python/test_v100_fullft_precision.py
View file

@ -13,6 +13,7 @@ unsloth/models/rl.py must then:
We execute the REAL template block extracted from rl.py source (no heavy unsloth
import) against mocked inputs. See issue #4082.
"""
from __future__ import annotations
import os
@ -35,32 +36,48 @@ def _extract_mixed_precision_code() -> str:
pytest.skip("mixed_precision template not found in rl.py")
body, k = [], start + 1
while lines[k].strip() != ")":
body.append(lines[k]); k += 1
body.append(lines[k])
k += 1
return eval("(\n" + "\n".join(body) + "\n)") # only string literals + comments
CODE = _extract_mixed_precision_code()
def _decide(dtype, *, bf16_supported, force_float32, full_finetuning,
mixed_precision, fp16, bf16):
def _decide(
dtype,
*,
bf16_supported,
force_float32,
full_finetuning,
mixed_precision,
fp16,
bf16,
):
"""Run the template block; return (args.fp16, args.bf16, ACCELERATE_MP, raised)."""
uz = types.ModuleType("unsloth_zoo"); uzu = types.ModuleType("unsloth_zoo.utils")
uz = types.ModuleType("unsloth_zoo")
uzu = types.ModuleType("unsloth_zoo.utils")
uzu._get_dtype = lambda x: x
sys.modules.setdefault("unsloth_zoo", uz); sys.modules["unsloth_zoo.utils"] = uzu
for k in ("UNSLOTH_FORCE_FLOAT32", "UNSLOTH_ENABLE_FULL_FINETUNING",
"UNSLOTH_MIXED_PRECISION", "ACCELERATE_MIXED_PRECISION"):
sys.modules.setdefault("unsloth_zoo", uz)
sys.modules["unsloth_zoo.utils"] = uzu
for k in (
"UNSLOTH_FORCE_FLOAT32",
"UNSLOTH_ENABLE_FULL_FINETUNING",
"UNSLOTH_MIXED_PRECISION",
"ACCELERATE_MIXED_PRECISION",
):
os.environ.pop(k, None)
os.environ["UNSLOTH_FORCE_FLOAT32"] = "1" if force_float32 else "0"
os.environ["UNSLOTH_ENABLE_FULL_FINETUNING"] = "1" if full_finetuning else "0"
os.environ["UNSLOTH_MIXED_PRECISION"] = mixed_precision
orig = torch.cuda.is_bf16_supported
torch.cuda.is_bf16_supported = lambda *a, **k: bf16_supported
args = types.SimpleNamespace(fp16=fp16, bf16=bf16, mixed_precision=None)
emb = types.SimpleNamespace(weight=types.SimpleNamespace(dtype=dtype))
args = types.SimpleNamespace(fp16 = fp16, bf16 = bf16, mixed_precision = None)
emb = types.SimpleNamespace(weight = types.SimpleNamespace(dtype = dtype))
model = types.SimpleNamespace(
config=types.SimpleNamespace(dtype=dtype, torch_dtype=dtype),
get_input_embeddings=lambda: emb)
config = types.SimpleNamespace(dtype = dtype, torch_dtype = dtype),
get_input_embeddings = lambda: emb,
)
raised = None
try:
exec(CODE, {"torch": torch, "os": os}, {"args": args, "model": model})
@ -74,17 +91,29 @@ def _decide(dtype, *, bf16_supported, force_float32, full_finetuning,
def test_v100_normal_fullft_fp16_explicit():
# Normal model, full FT (weights upcast to float32), V100, fp16=True.
fp16, bf16, amp, raised = _decide(
torch.float32, bf16_supported=False, force_float32=False,
full_finetuning=True, mixed_precision="float32", fp16=True, bf16=False)
torch.float32,
bf16_supported = False,
force_float32 = False,
full_finetuning = True,
mixed_precision = "float32",
fp16 = True,
bf16 = False,
)
assert raised is None
assert (fp16, bf16) == (True, False) # float32 weights + fp16 forward
assert (fp16, bf16) == (True, False) # float32 weights + fp16 forward
def test_v100_normal_fullft_precision_unset():
# Same, but user left precision unset -> must pick fp16, never bf16.
fp16, bf16, amp, raised = _decide(
torch.float32, bf16_supported=False, force_float32=False,
full_finetuning=True, mixed_precision="float32", fp16=False, bf16=False)
torch.float32,
bf16_supported = False,
force_float32 = False,
full_finetuning = True,
mixed_precision = "float32",
fp16 = False,
bf16 = False,
)
assert raised is None
assert (fp16, bf16) == (True, False)
assert amp == "fp16"
@ -93,8 +122,14 @@ def test_v100_normal_fullft_precision_unset():
def test_force_float32_model_fullft_is_pure_float32():
# FORCE_FLOAT32 model (Gemma3, gpt_oss, ...) in full FT -> pure float32, no autocast.
fp16, bf16, amp, raised = _decide(
torch.float32, bf16_supported=False, force_float32=True,
full_finetuning=True, mixed_precision="float32", fp16=True, bf16=False)
torch.float32,
bf16_supported = False,
force_float32 = True,
full_finetuning = True,
mixed_precision = "float32",
fp16 = True,
bf16 = False,
)
assert raised is None
assert (fp16, bf16) == (False, False)
assert amp in (None, "no")
@ -103,8 +138,14 @@ def test_force_float32_model_fullft_is_pure_float32():
def test_no_bf16_on_volta_in_auto_branch():
# bf16 model dtype but no bf16 HW, precision unset -> fp16, never bf16.
fp16, bf16, amp, raised = _decide(
torch.bfloat16, bf16_supported=False, force_float32=False,
full_finetuning=False, mixed_precision="float32", fp16=False, bf16=False)
torch.bfloat16,
bf16_supported = False,
force_float32 = False,
full_finetuning = False,
mixed_precision = "float32",
fp16 = False,
bf16 = False,
)
assert bf16 is False
@ -112,8 +153,14 @@ def test_bf16_gpu_unchanged_auto_branch():
# Regression guard: on a bf16 GPU, a float32 model with unset precision
# still selects bf16 autocast (behavior must not change for bf16 hardware).
fp16, bf16, amp, raised = _decide(
torch.float32, bf16_supported=True, force_float32=False,
full_finetuning=True, mixed_precision="float32", fp16=False, bf16=False)
torch.float32,
bf16_supported = True,
force_float32 = False,
full_finetuning = True,
mixed_precision = "float32",
fp16 = False,
bf16 = False,
)
assert raised is None
assert (fp16, bf16) == (False, True)
@ -121,6 +168,12 @@ def test_bf16_gpu_unchanged_auto_branch():
def test_genuine_bf16_model_with_fp16_still_raises():
# A real bfloat16 model on bf16 HW with fp16 requested is a genuine mismatch.
_, _, _, raised = _decide(
torch.bfloat16, bf16_supported=True, force_float32=False,
full_finetuning=False, mixed_precision="float32", fp16=True, bf16=False)
torch.bfloat16,
bf16_supported = True,
force_float32 = False,
full_finetuning = False,
mixed_precision = "float32",
fp16 = True,
bf16 = False,
)
assert raised == "TypeError"