borg/.github/workflows/ci.yml

# badge: https://github.com/borgbackup/borg/workflows/CI/badge.svg?branch=1.4-maint

name: CI

on:
  push:
    branches: [ 1.4-maint ]
    tags:
    - '1.*'
    paths:
    - '**.py'
    - '**.pyx'
    - '**.c'
    - '**.h'
    - '**.yml'
    - '**.toml'
    - '**.cfg'
    - '**.ini'
    - 'requirements.d/*'
    - '!docs/**'
  pull_request:
    branches: [ 1.4-maint ]
    paths:
    - '**.py'
    - '**.pyx'
    - '**.c'
    - '**.h'
    - '**.yml'
    - '**.toml'
    - '**.cfg'
    - '**.ini'
    - 'requirements.d/*'
    - '!docs/**'

jobs:
  lint:

    runs-on: ubuntu-24.04
    timeout-minutes: 5

    steps:
    - uses: actions/checkout@v4
    - uses: chartboost/ruff-action@v1


  asan_ubsan:

    runs-on: ubuntu-24.04
    timeout-minutes: 25
    needs: [lint]

    steps:
    - uses: actions/checkout@v4
      with:
        # Just fetching one commit is not enough for setuptools-scm, so we fetch all.
        fetch-depth: 0
        fetch-tags: true

    - name: Set up Python
      uses: actions/setup-python@v5
      with:
        python-version: '3.12'

    - name: Install system packages
      run: |
        sudo apt-get update
        sudo apt-get install -y pkg-config build-essential
        sudo apt-get install -y libssl-dev libacl1-dev libxxhash-dev liblz4-dev libzstd-dev

    - name: Install Python dependencies
      run: |
        python -m pip install --upgrade pip
        pip install -r requirements.d/development.txt

    - name: Build Borg with ASan/UBSan
      # Build the C/Cython extensions with AddressSanitizer and UndefinedBehaviorSanitizer enabled.
      # How this works:
      # - The -fsanitize=address,undefined flags inject runtime checks into our native code. If a bug is hit
      #   (e.g., buffer overflow, use-after-free, out-of-bounds, or undefined behavior), the sanitizer prints
      #   a detailed error report to stderr, including a stack trace, and forces the process to exit with
      #   non-zero status. In CI, this will fail the step/job so you will notice.
      # - ASAN_OPTIONS/UBSAN_OPTIONS configure the sanitizers' runtime behavior (see below for meanings).
      env:
        CFLAGS: "-O1 -g -fno-omit-frame-pointer -fsanitize=address,undefined"
        CXXFLAGS: "-O1 -g -fno-omit-frame-pointer -fsanitize=address,undefined"
        LDFLAGS: "-fsanitize=address,undefined"
        # ASAN_OPTIONS controls AddressSanitizer runtime tweaks:
        # - detect_leaks=0: Disable LeakSanitizer to avoid false positives with CPython/pymalloc in short-lived tests.
        # - strict_string_checks=1: Make invalid string operations (e.g., over-reads) more likely to be detected.
        # - check_initialization_order=1: Catch uses that depend on static initialization order (C++).
        # - detect_stack_use_after_return=1: Detect stack-use-after-return via stack poisoning (may increase overhead).
        ASAN_OPTIONS: "detect_leaks=0:strict_string_checks=1:check_initialization_order=1:detect_stack_use_after_return=1"
        # UBSAN_OPTIONS controls UndefinedBehaviorSanitizer runtime:
        # - print_stacktrace=1: Include a stack trace for UB reports to ease debugging.
        #   Note: UBSan is recoverable by default (process may continue after reporting). If you want CI to
        #   abort immediately and fail on the first UB, add `halt_on_error=1` (e.g., UBSAN_OPTIONS="print_stacktrace=1:halt_on_error=1").
        UBSAN_OPTIONS: "print_stacktrace=1"
        # PYTHONDEVMODE enables additional Python runtime checks and warnings.
        PYTHONDEVMODE: "1"
      run: pip install -e .

    - name: Run tests under sanitizers
      env:
        ASAN_OPTIONS: "detect_leaks=0:strict_string_checks=1:check_initialization_order=1:detect_stack_use_after_return=1"
        UBSAN_OPTIONS: "print_stacktrace=1"
        PYTHONDEVMODE: "1"
      # Ensure the ASan runtime is loaded first to avoid "ASan runtime does not come first" warnings.
      # We discover libasan/libubsan paths via gcc and preload them for the Python test process.
      # the remote tests are slow and likely won't find anything useful
      run: |
        set -euo pipefail
        export LD_PRELOAD="$(gcc -print-file-name=libasan.so):$(gcc -print-file-name=libubsan.so)"
        echo "Using LD_PRELOAD=$LD_PRELOAD"
        pytest -v --benchmark-skip -k "not remote"

  native_tests:

    needs: [lint]
    permissions:
      contents: read
      id-token: write
      attestations: write
    strategy:
      fail-fast: false
      # noinspection YAMLSchemaValidation
      matrix: >-
        ${{ fromJSON(
          github.event_name == 'pull_request' && '{
            "include": [
              {"os": "ubuntu-24.04", "python-version": "3.10", "toxenv": "py310-fuse2"},
              {"os": "ubuntu-24.04", "python-version": "3.14", "toxenv": "py314-fuse3"}
            ]
          }' || '{
            "include": [
              {"os": "ubuntu-24.04", "python-version": "3.10", "toxenv": "py310-fuse2"},
              {"os": "ubuntu-22.04", "python-version": "3.11", "toxenv": "py311-fuse3", "binary": "borg-linux-glibc235-x86_64-gh"},
              {"os": "ubuntu-22.04-arm", "python-version": "3.11", "toxenv": "py311-fuse3", "binary": "borg-linux-glibc235-arm64-gh"},
              {"os": "ubuntu-24.04", "python-version": "3.12", "toxenv": "py312-fuse3"},
              {"os": "ubuntu-24.04", "python-version": "3.13", "toxenv": "py313-fuse3"},
              {"os": "ubuntu-24.04", "python-version": "3.14", "toxenv": "py314-fuse3"},
              {"os": "macos-15-intel", "python-version": "3.11", "toxenv": "py311-none", "binary": "borg-macos-15-x86_64-gh"},
              {"os": "macos-15", "python-version": "3.11", "toxenv": "py311-none", "binary": "borg-macos-15-arm64-gh"}
            ]
          }'
        ) }}

    env:
      TOXENV: ${{ matrix.toxenv }}

    runs-on: ${{ matrix.os }}
    # macOS machines can be slow, if overloaded.
    timeout-minutes: 360

    steps:
    - uses: actions/checkout@v4
      with:
        # Just fetching one commit is not enough for setuptools-scm, so we fetch all
        fetch-depth: 0
        fetch-tags: true

    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}

    - name: Cache pip
      uses: actions/cache@v4
      with:
        path: ~/.cache/pip
        key: ${{ runner.os }}-pip-${{ hashFiles('requirements.d/development.txt') }}
        restore-keys: |
            ${{ runner.os }}-pip-
            ${{ runner.os }}-

    - name: Install Linux packages
      if: ${{ runner.os == 'Linux' }}
      run: |
        sudo apt-get update
        sudo apt-get install -y libssl-dev libacl1-dev liblz4-dev libzstd-dev pkg-config build-essential
        sudo apt-get install -y libxxhash-dev
        if [[ "$TOXENV" == *"fuse2"* ]]; then
          sudo apt-get install -y libfuse-dev fuse  # Required for Python llfuse module
        elif [[ "$TOXENV" == *"fuse3"* ]]; then
          sudo apt-get install -y libfuse3-dev fuse3  # Required for Python pyfuse3 module
        fi

    - name: Install macOS packages
      if: ${{ runner.os == 'macOS' }}
      run: |
        brew bundle install
        # the runner's unresolvable .local hostname takes macOS ~35s to give up on,
        # which borg paid per spawned process (import-time fqdn) - hours of test time,
        # see #9470. .local is mDNS territory, an /etc/hosts entry alone does not help.
        sudo scutil --set HostName borg-ci-mac
        echo "127.0.0.1 borg-ci-mac" | sudo tee -a /etc/hosts

    - name: Install Python requirements
      run: |
        python -m pip install --upgrade pip setuptools wheel
        pip install -r requirements.d/development.txt

    - name: Install BorgBackup
      run: |
        if [[ "$TOXENV" == *"fuse2"* ]]; then
          pip install -ve ".[llfuse]"
        elif [[ "$TOXENV" == *"fuse3"* ]]; then
          pip install -ve ".[pyfuse3]"
        else
          pip install -ve .
        fi

    - name: Build Borg fat binaries (${{ matrix.binary }})
      if: ${{ matrix.binary && startsWith(github.ref, 'refs/tags/') }}
      run: |
        pip install 'pyinstaller==6.14.2'
        mkdir -p dist/binary
        # Ensure locally built binaries in ./dist/binary are found during tox tests
        echo "$GITHUB_WORKSPACE/dist/binary" >> "$GITHUB_PATH"
        pyinstaller --clean --distpath=dist/binary scripts/borg.exe.spec

    - name: Smoke-test the built binary (${{ matrix.binary }})
      if: ${{ matrix.binary && startsWith(github.ref, 'refs/tags/') }}
      run: |
        pushd dist/binary
        echo "single-file binary"
        chmod +x borg.exe
        ./borg.exe -V
        echo "single-directory binary"
        chmod +x borg-dir/borg.exe
        ./borg-dir/borg.exe -V
        tar czf borg.tgz borg-dir
        popd
        echo "borg.exe binary in PATH"
        borg.exe -V

    - name: Prepare binaries (${{ matrix.binary }})
      if: ${{ matrix.binary && startsWith(github.ref, 'refs/tags/') }}
      run: |
        mkdir -p artifacts
        if [ -f dist/binary/borg.exe ]; then
          cp dist/binary/borg.exe artifacts/${{ matrix.binary }}
        fi
        if [ -f dist/binary/borg.tgz ]; then
          cp dist/binary/borg.tgz artifacts/${{ matrix.binary }}.tgz
        fi
        echo "binary files"
        ls -l artifacts/

    - name: Attest binaries provenance (${{ matrix.binary }})
      if: ${{ matrix.binary && startsWith(github.ref, 'refs/tags/') }}
      uses: actions/attest-build-provenance@v3
      with:
        subject-path: 'artifacts/*'

    - name: Upload binaries (${{ matrix.binary }})
      if: ${{ matrix.binary && startsWith(github.ref, 'refs/tags/') }}
      uses: actions/upload-artifact@v4
      with:
        name: ${{ matrix.binary }}
        path: artifacts/*
        if-no-files-found: error

    - name: Run pytest via tox
      run: |
        # Do not use fakeroot; run as root. Avoids the dreaded sporadic EISDIR failures; see #2482.
        #sudo -E bash -c "tox -e py"
        tox --skip-missing-interpreters

    - name: Upload coverage to Codecov
      uses: codecov/codecov-action@v4
      env:
        OS: ${{ runner.os }}
        python: ${{ matrix.python-version }}
      with:
        token: ${{ secrets.CODECOV_TOKEN }}
        env_vars: OS, python

  vm_tests:

    # Cross-OS tests running inside VMs, aligned with master branch structure.
    # Uses cross-platform-actions/action to run on FreeBSD, NetBSD, OpenBSD, Haiku.
    permissions:
      contents: read
      id-token: write
      attestations: write
    runs-on: ubuntu-24.04
    timeout-minutes: 90
    needs: [lint]
    continue-on-error: true
    strategy:
      fail-fast: false
      matrix:
        include:
          - os: freebsd
            version: '14.3'
            display_name: FreeBSD
            # Controls binary build and provenance attestation on tags
            do_binaries: true
            artifact_prefix: borg-freebsd-14-x86_64-gh
          - os: netbsd
            version: '11.0'
            display_name: NetBSD
            do_binaries: false
          - os: openbsd
            version: '7.9'
            display_name: OpenBSD
            do_binaries: false
            # extra RAM for the mfs-backed TMPDIR, see the openbsd test step
            memory: 12G
          - os: haiku
            version: 'r1beta5'
            display_name: Haiku
            do_binaries: false

    steps:
    - name: Check out repository
      uses: actions/checkout@v4
      with:
        fetch-depth: 0
        fetch-tags: true

    - name: Test on ${{ matrix.display_name }}
      id: cross_os
      uses: cross-platform-actions/action@v1.4.0
      env:
        DO_BINARIES: ${{ matrix.do_binaries }}
      with:
        operating_system: ${{ matrix.os }}
        version: ${{ matrix.version }}
        shell: bash
        # the default VM size (2 cpus) leaves half of the runner's 4 vcpus idle
        cpu_count: 4
        memory: ${{ matrix.memory || '8G' }}
        run: |
          set -euxo pipefail
          case "${{ matrix.os }}" in
            freebsd)
              # Ensure a proper hostname/FQDN is set (VMs may not have one by default)
              sudo -E /bin/sh -c 'grep -q "freebsd\.local" /etc/hosts || echo "127.0.0.1 freebsd.local freebsd" >> /etc/hosts'
              sudo -E hostname freebsd.local
              hostname
              export IGNORE_OSVERSION=yes
              sudo -E pkg update -f
              sudo -E pkg install -y xxhash liblz4 zstd pkgconf
              sudo -E pkg install -y fusefs-libs
              sudo -E kldload fusefs
              sudo -E sysctl vfs.usermount=1
              sudo -E chmod 666 /dev/fuse
              sudo -E pkg install -y rust
              sudo -E pkg install -y git
              sudo -E pkg install -y python310 py310-sqlite3
              sudo -E pkg install -y python311 py311-sqlite3
              sudo ln -sf /usr/local/bin/python3.11 /usr/local/bin/python3
              sudo ln -sf /usr/local/bin/python3.11 /usr/local/bin/python
              python -m venv .venv
              . .venv/bin/activate
              python -V
              pip -V
              python -m pip install --upgrade pip wheel
              pip install -r requirements.d/development.txt
              pip install -e ".[llfuse]"
              if [[ "${{ matrix.do_binaries }}" == "true" && "${{ startsWith(github.ref, 'refs/tags/') }}" == "true" ]]; then
                python -m pip install 'pyinstaller==6.14.2'
                mkdir -p dist/binary
                pyinstaller --clean --distpath=dist/binary scripts/borg.exe.spec
                pushd dist/binary
                echo "single-file binary"
                chmod +x borg.exe
                ./borg.exe -V
                echo "single-directory binary"
                chmod +x borg-dir/borg.exe
                ./borg-dir/borg.exe -V
                tar czf borg.tgz borg-dir
                popd
                mkdir -p artifacts
                if [ -f dist/binary/borg.exe ]; then
                  cp -v dist/binary/borg.exe artifacts/${{ matrix.artifact_prefix }}
                fi
                if [ -f dist/binary/borg.tgz ]; then
                  cp -v dist/binary/borg.tgz artifacts/${{ matrix.artifact_prefix }}.tgz
                fi
                echo "binary files"
                ls -l artifacts/
              fi
              export PATH="$(pwd)/dist/binary:$PATH"
              tox -e py311-fuse2
              ;;
            netbsd)
              # Ensure a proper hostname/FQDN is set (VMs may not have one by default)
              sudo -E /bin/sh -c 'grep -q "netbsd\.local" /etc/hosts || echo "127.0.0.1 netbsd.local netbsd" >> /etc/hosts'
              sudo -E hostname netbsd.local
              hostname
              arch="$(uname -m)"
              sudo -E mkdir -p /usr/pkg/etc/pkgin
              echo "https://ftp.NetBSD.org/pub/pkgsrc/packages/NetBSD/${arch}/11.0/All" | sudo tee /usr/pkg/etc/pkgin/repositories.conf > /dev/null
              sudo -E pkgin update
              sudo -E pkgin -y upgrade
              sudo -E pkgin -y install zstd lz4 xxhash git
              sudo -E pkgin -y install rust
              sudo -E pkgin -y install pkg-config
              # python 3.14 for coverage's fast sysmon core, see #9470
              sudo -E pkgin -y install py314-pip py314-virtualenv py314-tox
              sudo -E ln -sf /usr/pkg/bin/python3.14 /usr/pkg/bin/python3
              sudo -E ln -sf /usr/pkg/bin/pip3.14 /usr/pkg/bin/pip3
              sudo -E ln -sf /usr/pkg/bin/virtualenv-3.14 /usr/pkg/bin/virtualenv3
              sudo -E ln -sf /usr/pkg/bin/tox-3.14 /usr/pkg/bin/tox3
              # Ensure base system admin tools are on PATH for the non-root shell
              export PATH="/sbin:/usr/sbin:$PATH"
              # On the netbsd 11 VM, / is a fs with extended attributes,
              # while /tmp is a tmpfs without them - so don't use /tmp.
              export TMPDIR="/tmp_eafs"
              sudo -E mkdir -p ${TMPDIR}
              sudo -E chmod 1777 ${TMPDIR}
              touch ${TMPDIR}/testfile
              lsextattr user ${TMPDIR}/testfile && echo "[xattr] *** xattrs SUPPORTED on ${TMPDIR}! ***"
              tox3 -e py314-none
              ;;
            openbsd)
              # Put the temp tree (pip/cc build temps, pytest tmp dirs and the
              # borg test repos in them) on a memory-backed filesystem instead
              # of the slow FFS disk. Sized generously (the pytest tmp tree is
              # only cleaned up after the run); the VM gets 12G RAM for this.
              sudo mkdir -p /mfs
              # -O2: FFS2 format - mfs defaults to FFS1, whose 32 bit
              # timestamps silently wrap (breaks test_extract_y2261).
              sudo mount_mfs -O2 -s 6g swap /mfs
              sudo chmod 1777 /mfs
              export TMPDIR=/mfs

              sudo -E pkg_add xxhash lz4 zstd git
              sudo -E pkg_add rust
              sudo -E pkg_add openssl%3.5
              sudo -E pkg_add py3-pip py3-virtualenv py3-tox
              export BORG_OPENSSL_NAME=eopenssl35
              tox -e py313-none
              ;;
            haiku)
              pkgman refresh
              pkgman install -y git pkgconfig zstd lz4 xxhash
              pkgman install -y openssl3
              pkgman install -y rust_bin
              pkgman install -y python3.10
              pkgman install -y cffi
              pkgman install -y lz4_devel zstd_devel xxhash_devel openssl3_devel libffi_devel
              # there is no pkgman package for tox, so we install it into a venv
              python3 -m ensurepip --upgrade
              python3 -m pip install --upgrade pip wheel
              python3 -m venv .venv
              . .venv/bin/activate
              export PKG_CONFIG_PATH="/system/develop/lib/pkgconfig:/system/lib/pkgconfig:${PKG_CONFIG_PATH:-}"
              export BORG_LIBLZ4_PREFIX=/system/develop
              export BORG_LIBZSTD_PREFIX=/system/develop
              export BORG_LIBXXHASH_PREFIX=/system/develop
              export BORG_OPENSSL_PREFIX=/system/develop
              pip install -r requirements.d/development.txt
              pip install -e .
              # troubles with either tox or pytest xdist, so we run pytest manually:
              pytest -v -rs --benchmark-skip -k "not remote and not socket"
              ;;
          esac

    - name: Upload artifacts
      if: startsWith(github.ref, 'refs/tags/') && matrix.do_binaries
      uses: actions/upload-artifact@v4
      with:
        name: ${{ matrix.artifact_prefix }}
        path: artifacts/*
        if-no-files-found: ignore

    - name: Attest provenance
      if: startsWith(github.ref, 'refs/tags/') && matrix.do_binaries
      uses: actions/attest-build-provenance@v3
      with:
        subject-path: 'artifacts/*'