Free the failed calibration model before the basic-pipeline retry
In the sequential -> basic NVFP4 fallback, release the partially-processed model and clear the CUDA cache before loading a fresh copy, so the retry does not transiently hold two model copies on the GPU.
This commit is contained in:
parent
03be1d30f4
commit
f91ac9c1c2
1 changed files with 10 additions and 0 deletions
|
|
@ -200,6 +200,16 @@ def main():
|
|||
"retrying with the 'basic' pipeline (needs the full model to fit in memory).",
|
||||
flush = True,
|
||||
)
|
||||
# Free the partially-processed model (and the traceback frames pinning it) before
|
||||
# loading a fresh copy, so the fallback does not transiently hold two copies on GPU.
|
||||
import gc as _gc
|
||||
import torch as _torch
|
||||
|
||||
e = None
|
||||
del model
|
||||
_gc.collect()
|
||||
if _torch.cuda.is_available():
|
||||
_torch.cuda.empty_cache()
|
||||
model = _from_pretrained(auto_model, args.model, args.trust_remote_code)
|
||||
model.eval()
|
||||
oneshot(
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue