diff --git a/studio/backend/core/training/worker.py b/studio/backend/core/training/worker.py index ad421b8101..adb78c4704 100644 --- a/studio/backend/core/training/worker.py +++ b/studio/backend/core/training/worker.py @@ -2277,20 +2277,23 @@ def run_training_process( # Non-fatal: silently skipped if torch is not importable. if _hw.IS_ROCM: try: + import re as _re import torch as _torch_mem - import psutil as _psutil if _torch_mem.cuda.is_available(): - _vram_total = _torch_mem.cuda.get_device_properties(0).total_memory - _ram_total = _psutil.virtual_memory().total - _is_unified = _vram_total > (_ram_total * 0.5) + # iGPUs (gfx1150/gfx1151 Strix Halo, Strix Point, etc.) report + # names ending in a digit+M suffix ("AMD Radeon 890M"). + # Discrete cards use "RX NNNN [XT|XTX]" — no trailing M. + _dev_name = _torch_mem.cuda.get_device_properties(0).name + _is_unified = bool(_re.search(r'\d[Mm]\b', _dev_name)) _mem_fraction = 0.80 if _is_unified else 0.90 _torch_mem.cuda.set_per_process_memory_fraction(_mem_fraction) logger.info( "ROCm OOM guard: set_per_process_memory_fraction(%.2f) — " - "%s memory host", + "%s memory host (%s)", _mem_fraction, "unified" if _is_unified else "discrete", + _dev_name, ) except Exception as _oom_guard_err: logger.debug("Could not set GPU memory fraction: %s", _oom_guard_err)