ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update llama.py

Browse source
This commit is contained in:
Daniel Han 2025-03-21 17:08:00 -07:00
commit a7a223070b
1 changed files with 1 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

1
unsloth/models/llama.py
View file

@ -1575,6 +1575,7 @@ def unsloth_fast_generate(
kwargs["pad_token_id"] = kwargs.pop("pad_token_id", model_eos_token_id)
# Mixed precision autocast
print(args, kwargs)
with torch.inference_mode(), torch.autocast(device_type = "cuda", dtype = dtype):
output = self._old_generate(*args, **kwargs)
pass