diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py index 172ba36f63..ee82715306 100644 --- a/unsloth/import_fixes.py +++ b/unsloth/import_fixes.py @@ -97,6 +97,53 @@ class HidePrintMessage: return getattr(self._original_stream, name) +import contextlib +import ctypes + +try: + _libc = ctypes.CDLL(None) +except Exception: + _libc = None + + +@contextlib.contextmanager +def suppress_cuda_printf(): + """Suppress CUDA device-side printf by redirecting stdout/stderr fds to /dev/null. + + CUDA device printf (eg CUTLASS "Arch conditional MMA" errors on Blackwell) + writes to stdout fd 1 at the C level, bypassing Python sys.stdout entirely. + The existing HidePrintMessage filter on sys.stderr cannot catch these since + they go to a different fd at a different layer. This context manager redirects + both fd 1 and fd 2 at the OS level, syncs CUDA, then restores them. + """ + sys.stdout.flush() + sys.stderr.flush() + saved_fds = {} + try: + for fd in (1, 2): + saved_fds[fd] = os.dup(fd) + devnull = os.open(os.devnull, os.O_WRONLY) + os.dup2(devnull, fd) + os.close(devnull) + yield + finally: + try: + import torch + + if torch.cuda.is_available(): + torch.cuda.synchronize() + except Exception: + pass + if _libc is not None: + try: + _libc.fflush(None) + except Exception: + pass + for fd, saved in saved_fds.items(): + os.dup2(saved, fd) + os.close(saved) + + if not UNSLOTH_ENABLE_LOGGING: import sys diff --git a/unsloth/kernels/fp8.py b/unsloth/kernels/fp8.py index 0b073d59d6..a57f4ffb64 100644 --- a/unsloth/kernels/fp8.py +++ b/unsloth/kernels/fp8.py @@ -579,7 +579,14 @@ try: if Version(fbgemm_gpu.__version__) >= Version("1.4.0"): # We must manually confirm if blockwise FBGEMM works! # This check is a must for consumer grade GPUs which fail - if test_has_fbgemm(): + # Suppress CUDA device printf during probe -- on Blackwell (SM100) GPUs, + # FBGEMM's CUTLASS blockwise kernel (hardcoded SM90) fires thousands of + # "Arch conditional MMA" lines to stdout fd 1 before aborting. + from unsloth.import_fixes import suppress_cuda_printf + + with suppress_cuda_printf(): + _has_fbgemm = test_has_fbgemm() + if _has_fbgemm: os.environ["UNSLOTH_HAS_FBGEMM"] = "1" logger.info(f"Using fbgemm_gpu block quantized FP8 matmul") fp8_block_quant_linear = fp8_fbgemm_block_linear